From 9927ac782e135378f8088f03c63973500cd82b8b Mon Sep 17 00:00:00 2001 From: nym21 Date: Sun, 2 Aug 2026 14:26:25 +0200 Subject: [PATCH] global: mono repo --- Cargo.lock | 786 ++- Cargo.toml | 19 +- crates/brk_cli/Cargo.toml | 2 +- crates/brk_client/src/lib.rs | 6266 +++++++++++++---- .../src/distribution/addr/indexes/any.rs | 13 +- .../src/distribution/block/cache/addr.rs | 26 +- .../src/distribution/block/utxo/inputs.rs | 24 +- .../src/distribution/block/utxo/outputs.rs | 24 +- .../src/distribution/compute/block_loop.rs | 16 +- .../src/distribution/compute/readers.rs | 83 +- crates/brk_computer/src/distribution/vecs.rs | 45 +- .../brk_computer/src/inputs/spent/compute.rs | 5 +- crates/brk_computer/src/investing/import.rs | 5 +- crates/brk_computer/src/lib.rs | 1 + crates/brk_computer/src/mining/import.rs | 4 +- .../src/mining/rewards/compute.rs | 7 +- .../brk_computer/src/mining/rewards/import.rs | 11 +- .../src/outputs/by_type/compute.rs | 9 +- .../brk_computer/src/outputs/spent/compute.rs | 7 + crates/brk_computer/src/pools/mod.rs | 51 +- crates/brk_computer/src/pools/pool_heights.rs | 4 +- crates/brk_computer/src/price/compute.rs | 12 +- .../src/transactions/patterns/compute.rs | 6 +- crates/brk_indexer/src/lib.rs | 7 +- .../src/processor/transaction/mod.rs | 6 +- crates/brk_indexer/src/processor/txin/mod.rs | 8 +- .../src/processor/txin/resolver.rs | 297 +- .../brk_indexer/src/processor/txin/source.rs | 2 +- crates/brk_indexer/src/readers.rs | 16 +- crates/brk_indexer/src/stores.rs | 13 +- crates/brk_indexer/src/vecs/addrs.rs | 18 +- crates/brk_indexer/src/vecs/scripts.rs | 4 +- crates/brk_oracle/examples/determinism.rs | 7 +- crates/brk_oracle/examples/dump_hist.rs | 7 +- crates/brk_oracle/examples/experiment.rs | 7 +- crates/brk_oracle/examples/report.rs | 7 +- crates/brk_oracle/examples/report_from.rs | 7 +- crates/brk_query/Cargo.toml | 3 +- crates/brk_query/src/impl/addr/stats.rs | 9 +- crates/brk_query/src/impl/addr/txs.rs | 3 +- crates/brk_query/src/impl/addr/utxos.rs | 7 +- crates/brk_query/src/impl/block/txs.rs | 5 +- crates/brk_query/src/impl/cpfp/confirmed.rs | 13 +- crates/brk_query/src/impl/mempool.rs | 10 +- crates/brk_query/src/impl/tx.rs | 17 +- crates/brk_store/Cargo.toml | 3 + crates/brk_store/src/any.rs | 9 +- crates/brk_store/src/lib.rs | 135 +- crates/brk_store/src/meta.rs | 8 +- crates/brk_store/tests/owned_ingest.rs | 59 + crates/brk_types/src/addr_index_outpoint.rs | 45 +- crates/brk_types/src/addr_index_tx_index.rs | 5 + crates/brk_types/src/pools.rs | 5 +- crates/fjall/.config/nextest.toml | 2 + crates/fjall/.gitignore | 24 + crates/fjall/.rustfmt.toml | 3 + crates/fjall/.vscode/settings.json | 3 + crates/fjall/CHANGELOG.md | 50 + crates/fjall/CONTRIBUTING.md | 11 + crates/fjall/Cargo.toml | 42 + crates/fjall/LICENSE-APACHE | 176 + crates/fjall/LICENSE-MIT | 21 + crates/fjall/README.md | 207 + crates/fjall/commit.nu | 64 + crates/fjall/compile_examples.mjs | 60 + crates/fjall/kawaii.png | Bin 0 -> 92194 bytes crates/fjall/logo.png | Bin 0 -> 17908 bytes crates/fjall/renovate.json | 6 + crates/fjall/src/batch/item.rs | 73 + crates/fjall/src/batch/mod.rs | 181 + crates/fjall/src/builder.rs | 190 + crates/fjall/src/compaction/mod.rs | 17 + crates/fjall/src/compaction/worker.rs | 60 + crates/fjall/src/db.rs | 921 +++ crates/fjall/src/db_config.rs | 95 + crates/fjall/src/db_test.rs | 244 + crates/fjall/src/drop.rs | 25 + crates/fjall/src/error.rs | 83 + crates/fjall/src/file.rs | 35 + crates/fjall/src/flush/manager.rs | 44 + crates/fjall/src/flush/mod.rs | 9 + crates/fjall/src/flush/task.rs | 15 + crates/fjall/src/flush/worker.rs | 42 + crates/fjall/src/guard.rs | 153 + crates/fjall/src/ingestion.rs | 100 + crates/fjall/src/iter.rs | 40 + crates/fjall/src/journal/batch_reader.rs | 216 + crates/fjall/src/journal/entry.rs | 306 + crates/fjall/src/journal/error.rs | 30 + crates/fjall/src/journal/manager.rs | 186 + crates/fjall/src/journal/mod.rs | 134 + crates/fjall/src/journal/reader.rs | 78 + crates/fjall/src/journal/recovery.rs | 91 + crates/fjall/src/journal/test.rs | 488 ++ crates/fjall/src/journal/writer.rs | 380 + .../fjall/src/keyspace/config/block_size.rs | 55 + .../fjall/src/keyspace/config/compression.rs | 61 + crates/fjall/src/keyspace/config/filter.rs | 117 + .../fjall/src/keyspace/config/hash_ratio.rs | 55 + crates/fjall/src/keyspace/config/mod.rs | 20 + crates/fjall/src/keyspace/config/pinning.rs | 54 + .../src/keyspace/config/restart_interval.rs | 54 + crates/fjall/src/keyspace/mod.rs | 1178 ++++ crates/fjall/src/keyspace/name.rs | 14 + crates/fjall/src/keyspace/options.rs | 749 ++ crates/fjall/src/keyspace/test.rs | 75 + crates/fjall/src/keyspace/write_delay.rs | 16 + crates/fjall/src/lib.rs | 180 + crates/fjall/src/locked_file.rs | 112 + crates/fjall/src/meta_keyspace.rs | 483 ++ crates/fjall/src/path.rs | 10 + crates/fjall/src/poison.rs | 41 + crates/fjall/src/readable.rs | 301 + crates/fjall/src/recovery.rs | 270 + crates/fjall/src/snapshot.rs | 105 + crates/fjall/src/snapshot_nonce.rs | 38 + crates/fjall/src/snapshot_tracker.rs | 405 ++ crates/fjall/src/stats.rs | 20 + crates/fjall/src/supervisor.rs | 70 + crates/fjall/src/tx/mod.rs | 7 + .../src/tx/optimistic/conflict_manager.rs | 197 + crates/fjall/src/tx/optimistic/keyspace.rs | 502 ++ crates/fjall/src/tx/optimistic/mod.rs | 184 + crates/fjall/src/tx/optimistic/oracle.rs | 121 + crates/fjall/src/tx/optimistic/write_tx.rs | 891 +++ crates/fjall/src/tx/single_writer/keyspace.rs | 469 ++ crates/fjall/src/tx/single_writer/mod.rs | 189 + crates/fjall/src/tx/single_writer/write_tx.rs | 375 + crates/fjall/src/tx/write_tx.rs | 404 ++ crates/fjall/src/version.rs | 151 + crates/fjall/src/worker_pool.rs | 270 + crates/fjall/src/write_buffer_manager.rs | 76 + .../v1_keyspace/partitions/a/config | Bin 0 -> 16 bytes .../v1_keyspace/partitions/a/levels | Bin 0 -> 45 bytes .../v1_keyspace/partitions/a/segments/0 | Bin 0 -> 856 bytes .../v1_keyspace/partitions/a/version | Bin 0 -> 5 bytes .../v1_keyspace/partitions/b/config | Bin 0 -> 16 bytes .../v1_keyspace/partitions/b/levels | Bin 0 -> 45 bytes .../v1_keyspace/partitions/b/segments/1 | Bin 0 -> 616 bytes .../v1_keyspace/partitions/b/version | Bin 0 -> 5 bytes .../v1_keyspace/partitions/c/config | Bin 0 -> 16 bytes .../v1_keyspace/partitions/c/levels | Bin 0 -> 37 bytes .../v1_keyspace/partitions/c/version | Bin 0 -> 5 bytes crates/fjall/test_fixture/v1_keyspace/version | Bin 0 -> 5 bytes .../partitions/a/config | Bin 0 -> 16 bytes .../partitions/a/levels | Bin 0 -> 45 bytes .../partitions/a/segments/0 | Bin 0 -> 856 bytes .../partitions/a/version | Bin 0 -> 5 bytes .../partitions/b/config | Bin 0 -> 16 bytes .../partitions/b/levels | Bin 0 -> 45 bytes .../partitions/b/segments/1 | Bin 0 -> 616 bytes .../partitions/b/version | Bin 0 -> 5 bytes .../partitions/c/config | Bin 0 -> 16 bytes .../partitions/c/levels | Bin 0 -> 37 bytes .../partitions/c/version | Bin 0 -> 5 bytes .../v1_keyspace_corrupt_journal/version | Bin 0 -> 5 bytes .../fjall/test_fixture/v2_keyspace/journals/2 | Bin 0 -> 56240 bytes .../v2_keyspace/partitions/default1/config | Bin 0 -> 30 bytes .../v2_keyspace/partitions/default1/levels | Bin 0 -> 41 bytes .../v2_keyspace/partitions/default1/manifest | Bin 0 -> 7 bytes .../partitions/default1/segments/0 | Bin 0 -> 732 bytes .../partitions/default2/blobs/.vlog | 1 + .../partitions/default2/blobs/segments/0 | Bin 0 -> 1337 bytes .../partitions/default2/blobs/vlog_manifest | Bin 0 -> 16 bytes .../v2_keyspace/partitions/default2/config | Bin 0 -> 44 bytes .../v2_keyspace/partitions/default2/levels | Bin 0 -> 41 bytes .../v2_keyspace/partitions/default2/manifest | Bin 0 -> 7 bytes .../partitions/default2/segments/0 | Bin 0 -> 576 bytes crates/fjall/test_fixture/v2_keyspace/version | 1 + .../v2_keyspace_corrupt_journal/journals/2 | Bin 0 -> 16777216 bytes .../partitions/default1/config | Bin 0 -> 30 bytes .../partitions/default1/levels | Bin 0 -> 41 bytes .../partitions/default1/manifest | Bin 0 -> 7 bytes .../partitions/default1/segments/0 | Bin 0 -> 732 bytes .../partitions/default2/blobs/.vlog | 1 + .../partitions/default2/blobs/segments/0 | Bin 0 -> 1337 bytes .../partitions/default2/blobs/vlog_manifest | Bin 0 -> 16 bytes .../partitions/default2/config | Bin 0 -> 44 bytes .../partitions/default2/levels | Bin 0 -> 41 bytes .../partitions/default2/manifest | Bin 0 -> 7 bytes .../partitions/default2/segments/0 | Bin 0 -> 576 bytes .../v2_keyspace_corrupt_journal/version | 1 + .../journals/0.sealed | Bin 0 -> 46 bytes .../journals/1.sealed | Bin 0 -> 46 bytes .../v2_sealed_journal_shenanigans/journals/2 | Bin 0 -> 46 bytes .../v2_sealed_journal_shenanigans/journals/3 | 0 .../partitions/default/config | Bin 0 -> 30 bytes .../partitions/default/levels | Bin 0 -> 33 bytes .../partitions/default/manifest | Bin 0 -> 7 bytes .../v2_sealed_journal_shenanigans/version | 1 + crates/fjall/tests/batch.rs | 70 + crates/fjall/tests/batch_recovery.rs | 21 + crates/fjall/tests/blob_kv_simple.rs | 63 + crates/fjall/tests/clear_dirty_read.rs | 35 + crates/fjall/tests/compaction_filter.rs | 158 + crates/fjall/tests/db_lock.rs | 23 + crates/fjall/tests/db_open.rs | 34 + crates/fjall/tests/fifo_dirty_read.rs | 31 + crates/fjall/tests/format_v4.rs | 41 + crates/fjall/tests/ingest_recovery.rs | 73 + crates/fjall/tests/journal_large_value.rs | 55 + crates/fjall/tests/keyspace_clear.rs | 84 + crates/fjall/tests/keyspace_iter_lifetime.rs | 34 + crates/fjall/tests/keyspace_recover.rs | 290 + crates/fjall/tests/keyspace_snapshot.rs | 108 + .../fjall/tests/keyspace_v1_load_fixture.rs | 30 + .../fjall/tests/keyspace_v2_load_fixture.rs | 32 + crates/fjall/tests/memtable_recover.rs | 47 + crates/fjall/tests/prefix_complex.rs | 45 + .../tests/recover_from_different_folder.rs | 35 + crates/fjall/tests/recovery_journal_mac.rs | 36 + crates/fjall/tests/recovery_keyspaces_mac.rs | 36 + crates/fjall/tests/seqno_recovery.rs | 155 + crates/fjall/tests/standard_keyspace.rs | 62 + crates/fjall/tests/tx_ryow.rs | 31 + crates/fjall/tests/tx_ryow_snapshot.rs | 59 + crates/fjall/tests/write_buffer_size.rs | 58 + crates/fjall/tests/write_during_read.rs | 25 + crates/fjall/tests/write_tx.rs | 20 + crates/lsm-tree/.config/nextest.toml | 2 + crates/lsm-tree/.gitignore | 26 + crates/lsm-tree/.rustfmt.toml | 3 + crates/lsm-tree/CONTRIBUTING.md | 11 + crates/lsm-tree/Cargo.toml | 97 + crates/lsm-tree/LICENSE-APACHE | 176 + crates/lsm-tree/LICENSE-MIT | 21 + crates/lsm-tree/README.md | 83 + crates/lsm-tree/UNSAFE.md | 33 + crates/lsm-tree/benches/block.rs | 62 + crates/lsm-tree/benches/bloom.rs | 107 + crates/lsm-tree/benches/fd_table.rs | 101 + crates/lsm-tree/benches/level_manifest.rs | 33 + crates/lsm-tree/benches/memtable.rs | 115 + crates/lsm-tree/benches/merge.rs | 77 + crates/lsm-tree/benches/tree.rs | 412 ++ crates/lsm-tree/logo.png | Bin 0 -> 19998 bytes crates/lsm-tree/renovate.json | 6 + crates/lsm-tree/src/abstract_tree.rs | 609 ++ crates/lsm-tree/src/any_tree.rs | 17 + crates/lsm-tree/src/blob_tree/gc.rs | 307 + crates/lsm-tree/src/blob_tree/handle.rs | 32 + crates/lsm-tree/src/blob_tree/ingest.rs | 274 + crates/lsm-tree/src/blob_tree/mod.rs | 631 ++ crates/lsm-tree/src/cache.rs | 161 + crates/lsm-tree/src/checksum.rs | 96 + crates/lsm-tree/src/coding.rs | 30 + crates/lsm-tree/src/compaction/drop_range.rs | 100 + crates/lsm-tree/src/compaction/fifo.rs | 310 + crates/lsm-tree/src/compaction/filter.rs | 278 + crates/lsm-tree/src/compaction/flavour.rs | 543 ++ crates/lsm-tree/src/compaction/leveled/mod.rs | 579 ++ .../lsm-tree/src/compaction/leveled/test.rs | 177 + crates/lsm-tree/src/compaction/maintenance.rs | 179 + crates/lsm-tree/src/compaction/major.rs | 57 + crates/lsm-tree/src/compaction/mod.rs | 98 + crates/lsm-tree/src/compaction/movedown.rs | 38 + crates/lsm-tree/src/compaction/pulldown.rs | 42 + .../src/compaction/state/hidden_set.rs | 47 + crates/lsm-tree/src/compaction/state/mod.rs | 90 + crates/lsm-tree/src/compaction/stream.rs | 835 +++ crates/lsm-tree/src/compaction/tiered.rs | 343 + crates/lsm-tree/src/compaction/worker.rs | 960 +++ crates/lsm-tree/src/compression.rs | 94 + crates/lsm-tree/src/config/block_size.rs | 44 + crates/lsm-tree/src/config/compression.rs | 52 + crates/lsm-tree/src/config/filter.rs | 66 + crates/lsm-tree/src/config/hash_ratio.rs | 44 + crates/lsm-tree/src/config/mod.rs | 472 ++ crates/lsm-tree/src/config/pinning.rs | 50 + .../lsm-tree/src/config/restart_interval.rs | 46 + crates/lsm-tree/src/descriptor_table.rs | 73 + crates/lsm-tree/src/double_ended_peekable.rs | 197 + crates/lsm-tree/src/error.rs | 94 + crates/lsm-tree/src/file.rs | 232 + crates/lsm-tree/src/file_accessor.rs | 70 + crates/lsm-tree/src/format_version.rs | 50 + crates/lsm-tree/src/hash.rs | 33 + crates/lsm-tree/src/ingestion.rs | 126 + crates/lsm-tree/src/iter_guard.rs | 64 + crates/lsm-tree/src/key.rs | 107 + crates/lsm-tree/src/key_range.rs | 343 + crates/lsm-tree/src/lib.rs | 207 + crates/lsm-tree/src/manifest.rs | 93 + crates/lsm-tree/src/memtable/mod.rs | 381 + crates/lsm-tree/src/merge.rs | 176 + crates/lsm-tree/src/metrics.rs | 212 + crates/lsm-tree/src/mvcc_stream.rs | 688 ++ crates/lsm-tree/src/path.rs | 11 + crates/lsm-tree/src/range.rs | 298 + crates/lsm-tree/src/run_reader.rs | 325 + crates/lsm-tree/src/run_scanner.rs | 154 + crates/lsm-tree/src/seqno.rs | 106 + crates/lsm-tree/src/slice/mod.rs | 260 + crates/lsm-tree/src/slice/slice_bytes/mod.rs | 115 + .../lsm-tree/src/slice/slice_default/mod.rs | 78 + crates/lsm-tree/src/slice_windows.rs | 94 + crates/lsm-tree/src/stop_signal.rs | 19 + .../src/table/block/binary_index/builder.rs | 55 + .../src/table/block/binary_index/mod.rs | 9 + .../src/table/block/binary_index/reader.rs | 49 + crates/lsm-tree/src/table/block/decoder.rs | 558 ++ crates/lsm-tree/src/table/block/encoder.rs | 197 + .../src/table/block/hash_index/builder.rs | 124 + .../src/table/block/hash_index/mod.rs | 143 + .../src/table/block/hash_index/reader.rs | 59 + crates/lsm-tree/src/table/block/header.rs | 215 + crates/lsm-tree/src/table/block/mod.rs | 231 + crates/lsm-tree/src/table/block/offset.rs | 27 + crates/lsm-tree/src/table/block/trailer.rs | 198 + crates/lsm-tree/src/table/block/type.rs | 36 + crates/lsm-tree/src/table/block_index/full.rs | 61 + crates/lsm-tree/src/table/block_index/iter.rs | 47 + crates/lsm-tree/src/table/block_index/mod.rs | 131 + .../src/table/block_index/two_level.rs | 236 + .../src/table/block_index/volatile.rs | 176 + crates/lsm-tree/src/table/data_block/iter.rs | 191 + .../src/table/data_block/iter_test.rs | 1280 ++++ crates/lsm-tree/src/table/data_block/mod.rs | 1378 ++++ .../src/table/filter/bit_array/builder.rs | 71 + .../src/table/filter/bit_array/mod.rs | 9 + .../src/table/filter/bit_array/reader.rs | 41 + crates/lsm-tree/src/table/filter/block.rs | 25 + .../table/filter/blocked_bloom/builder._rs | 169 + .../src/table/filter/blocked_bloom/mod._rs | 193 + crates/lsm-tree/src/table/filter/mod.rs | 115 + .../table/filter/standard_bloom/builder.rs | 188 + .../src/table/filter/standard_bloom/mod.rs | 305 + crates/lsm-tree/src/table/id.rs | 44 + .../src/table/index_block/block_handle.rs | 266 + crates/lsm-tree/src/table/index_block/iter.rs | 672 ++ crates/lsm-tree/src/table/index_block/mod.rs | 128 + crates/lsm-tree/src/table/inner.rs | 97 + crates/lsm-tree/src/table/iter.rs | 414 ++ crates/lsm-tree/src/table/meta.rs | 225 + crates/lsm-tree/src/table/mod.rs | 686 ++ crates/lsm-tree/src/table/multi_writer.rs | 350 + crates/lsm-tree/src/table/regions.rs | 77 + crates/lsm-tree/src/table/scanner.rs | 93 + crates/lsm-tree/src/table/tests.rs | 1474 ++++ crates/lsm-tree/src/table/util.rs | 224 + .../lsm-tree/src/table/writer/filter/full.rs | 96 + .../lsm-tree/src/table/writer/filter/mod.rs | 41 + .../src/table/writer/filter/partitioned.rs | 209 + .../lsm-tree/src/table/writer/index/full.rs | 86 + crates/lsm-tree/src/table/writer/index/mod.rs | 32 + .../src/table/writer/index/partitioned.rs | 202 + crates/lsm-tree/src/table/writer/meta.rs | 65 + crates/lsm-tree/src/table/writer/mod.rs | 678 ++ crates/lsm-tree/src/time.rs | 35 + crates/lsm-tree/src/tree/ingest.rs | 387 + crates/lsm-tree/src/tree/inner.rs | 115 + crates/lsm-tree/src/tree/mod.rs | 1228 ++++ crates/lsm-tree/src/tree/sealed.rs | 41 + crates/lsm-tree/src/util.rs | 173 + crates/lsm-tree/src/value.rs | 143 + crates/lsm-tree/src/value_type.rs | 59 + crates/lsm-tree/src/version/blob_file_list.rs | 60 + crates/lsm-tree/src/version/mod.rs | 704 ++ crates/lsm-tree/src/version/optimize.rs | 161 + crates/lsm-tree/src/version/persist.rs | 80 + crates/lsm-tree/src/version/recovery.rs | 160 + crates/lsm-tree/src/version/run.rs | 422 ++ crates/lsm-tree/src/version/super_version.rs | 370 + crates/lsm-tree/src/vlog/accessor.rs | 56 + crates/lsm-tree/src/vlog/blob_file/merge.rs | 203 + crates/lsm-tree/src/vlog/blob_file/meta.rs | 186 + crates/lsm-tree/src/vlog/blob_file/mod.rs | 156 + .../src/vlog/blob_file/multi_writer.rs | 242 + crates/lsm-tree/src/vlog/blob_file/reader.rs | 161 + crates/lsm-tree/src/vlog/blob_file/scanner.rs | 169 + crates/lsm-tree/src/vlog/blob_file/writer.rs | 247 + crates/lsm-tree/src/vlog/handle.rs | 49 + crates/lsm-tree/src/vlog/mod.rs | 150 + crates/lsm-tree/test_fixture/v1_tree/config | Bin 0 -> 16 bytes crates/lsm-tree/test_fixture/v1_tree/levels | Bin 0 -> 45 bytes .../lsm-tree/test_fixture/v1_tree/segments/0 | Bin 0 -> 832 bytes crates/lsm-tree/test_fixture/v1_tree/version | Bin 0 -> 5 bytes .../test_fixture/v1_tree_corrupt/config | Bin 0 -> 16 bytes .../test_fixture/v1_tree_corrupt/levels | Bin 0 -> 45 bytes .../test_fixture/v1_tree_corrupt/segments/0 | Bin 0 -> 832 bytes .../test_fixture/v1_tree_corrupt/version | Bin 0 -> 5 bytes crates/lsm-tree/tests/blob_compression.rs | 99 + crates/lsm-tree/tests/blob_empty_flush.rs | 30 + crates/lsm-tree/tests/blob_fifo_limit.rs | 28 + .../tests/blob_file_full_file_checksum.rs | 121 + crates/lsm-tree/tests/blob_flush_gc_stats.rs | 71 + crates/lsm-tree/tests/blob_gc_watermark.rs | 105 + crates/lsm-tree/tests/blob_ingest.rs | 61 + crates/lsm-tree/tests/blob_ingest_relink.rs | 66 + .../blob_major_compact_drop_dead_files.rs | 100 + .../tests/blob_major_compact_gc_stats.rs | 205 + .../tests/blob_major_compact_relink.rs | 69 + .../tests/blob_major_compact_relocation.rs | 326 + .../blob_major_compact_relocation_recovery.rs | 130 + crates/lsm-tree/tests/blob_nuke_gc_stats.rs | 110 + crates/lsm-tree/tests/blob_pinned_fd.rs | 58 + .../lsm-tree/tests/blob_recover_gc_stats.rs | 82 + crates/lsm-tree/tests/blob_recovery.rs | 49 + .../tests/blob_register_table_rotation.rs | 135 + crates/lsm-tree/tests/blob_sep_threshold.rs | 30 + crates/lsm-tree/tests/blob_simple.rs | 77 + .../lsm-tree/tests/blob_tree_guarded_size.rs | 25 + .../lsm-tree/tests/blob_tree_reload_blob.rs | 144 + crates/lsm-tree/tests/compaction_filter.rs | 199 + .../compaction_filter_gc_stats_change.rs | 263 + .../compaction_filter_gc_stats_destroy.rs | 90 + .../compaction_filter_gc_stats_remove.rs | 90 + .../compaction_filter_gc_stats_remove_weak.rs | 90 + .../tests/compaction_filter_snapshot.rs | 56 + .../lsm-tree/tests/compaction_filter_ttl.rs | 72 + .../tests/compaction_readers_grouping.rs | 64 + crates/lsm-tree/tests/dirty_snapshot.rs | 24 + .../lsm-tree/tests/expect_point_read_hits.rs | 48 + crates/lsm-tree/tests/fifo_ttl.rs | 134 + crates/lsm-tree/tests/guard_if.rs | 71 + .../lsm-tree/tests/ingest_dirty_snapshot.rs | 23 + crates/lsm-tree/tests/ingestion_api.rs | 368 + crates/lsm-tree/tests/ingestion_invariants.rs | 181 + crates/lsm-tree/tests/ingestion_seqno.rs | 25 + crates/lsm-tree/tests/latest_snapshot.rs | 24 + crates/lsm-tree/tests/leveled_trivial_move.rs | 60 + crates/lsm-tree/tests/memtable_id.rs | 40 + crates/lsm-tree/tests/model_1.rs | 43 + crates/lsm-tree/tests/model_2.rs | 48 + crates/lsm-tree/tests/model_3.rs | 70 + crates/lsm-tree/tests/model_4.rs | 112 + crates/lsm-tree/tests/model_5.rs | 87 + crates/lsm-tree/tests/model_6.rs | 1293 ++++ crates/lsm-tree/tests/multi_trees.rs | 69 + crates/lsm-tree/tests/mvcc_slab.rs | 77 + .../lsm-tree/tests/recover_cleanup_orphans.rs | 90 + .../tests/recover_from_different_folder.rs | 74 + .../lsm-tree/tests/recovery_mac_ds_store.rs | 36 + .../lsm-tree/tests/recovery_mac_underscore.rs | 36 + crates/lsm-tree/tests/regression_286.rs | 32 + crates/lsm-tree/tests/snapshot_compact.rs | 61 + crates/lsm-tree/tests/snapshot_len.rs | 41 + crates/lsm-tree/tests/snapshot_point_read.rs | 189 + crates/lsm-tree/tests/snapshot_zombie.rs | 103 + crates/lsm-tree/tests/static_iterators.rs | 363 + .../tests/table_full_file_checksum.rs | 115 + crates/lsm-tree/tests/table_point_reads.rs | 172 + crates/lsm-tree/tests/table_range.rs | 157 + crates/lsm-tree/tests/table_range_oob.rs | 87 + crates/lsm-tree/tests/table_remove_weak.rs | 28 + .../lsm-tree/tests/table_weak_tombstones.rs | 75 + crates/lsm-tree/tests/tree_approx_len.rs | 212 + crates/lsm-tree/tests/tree_bulk_ingest.rs | 205 + crates/lsm-tree/tests/tree_clear.rs | 43 + crates/lsm-tree/tests/tree_clear_snapshot.rs | 30 + crates/lsm-tree/tests/tree_count.rs | 154 + crates/lsm-tree/tests/tree_delete_loop.rs | 94 + .../tests/tree_different_block_size.rs | 72 + crates/lsm-tree/tests/tree_disjoint_iter.rs | 74 + .../tests/tree_disjoint_point_read.rs | 71 + crates/lsm-tree/tests/tree_disjoint_prefix.rs | 70 + crates/lsm-tree/tests/tree_disjoint_range.rs | 106 + crates/lsm-tree/tests/tree_drop_range.rs | 277 + crates/lsm-tree/tests/tree_filter_hit_rate.rs | 86 + crates/lsm-tree/tests/tree_flush_eviction.rs | 138 + crates/lsm-tree/tests/tree_guarded_range.rs | 73 + crates/lsm-tree/tests/tree_kv.rs | 160 + crates/lsm-tree/tests/tree_l0_point_read.rs | 48 + crates/lsm-tree/tests/tree_l0_range.rs | 51 + .../lsm-tree/tests/tree_major_compaction.rs | 56 + crates/lsm-tree/tests/tree_mvcc_simple.rs | 115 + .../tests/tree_non_disjoint_point_read.rs | 61 + crates/lsm-tree/tests/tree_range.rs | 133 + .../tests/tree_range_memtable_only.rs | 80 + crates/lsm-tree/tests/tree_recover_counter.rs | 36 + .../tests/tree_recover_large_value.rs | 33 + .../lsm-tree/tests/tree_recovery_versions.rs | 48 + crates/lsm-tree/tests/tree_reload.rs | 264 + crates/lsm-tree/tests/tree_reload_pwd.rs | 50 + .../lsm-tree/tests/tree_sealed_shadowing.rs | 41 + crates/lsm-tree/tests/tree_seqno.rs | 64 + crates/lsm-tree/tests/tree_shadowing.rs | 381 + crates/lsm-tree/tests/tree_trace.rs | 34 + crates/lsm-tree/tests/tree_v1_load_fixture.rs | 34 + crates/lsm-tree/tests/tree_weak_delete.rs | 167 + .../tests/tree_weak_delete_eviction.rs | 30 + .../lsm-tree/tests/tree_weak_delete_queue.rs | 107 + crates/lsm-tree/tests/tree_write_read.rs | 59 + crates/quickmatch/.gitignore | 7 + crates/quickmatch/Cargo.toml | 14 + crates/quickmatch/docs/CHANGELOG.md | 161 + crates/quickmatch/docs/LICENSE.md | 21 + crates/quickmatch/docs/README.md | 95 + crates/quickmatch/examples/autocomplete.rs | 59 + crates/quickmatch/examples/colon_test.rs | 29 + crates/quickmatch/examples/compare.rs | 111 + crates/quickmatch/src/config.rs | 85 + crates/quickmatch/src/lib.rs | 381 + crates/rawdb/Cargo.toml | 26 + crates/rawdb/README.md | 96 + crates/rawdb/src/disk_usage.rs | 50 + crates/rawdb/src/error.rs | 88 + crates/rawdb/src/hints.rs | 19 + crates/rawdb/src/hole_punch.rs | 108 + crates/rawdb/src/layout.rs | 226 + crates/rawdb/src/lib.rs | 666 ++ crates/rawdb/src/mmap.rs | 24 + crates/rawdb/src/reader.rs | 86 + crates/rawdb/src/region.rs | 457 ++ crates/rawdb/src/region_metadata.rs | 258 + crates/rawdb/src/region_state.rs | 56 + crates/rawdb/src/regions.rs | 212 + crates/rawdb/tests/mod.rs | 2077 ++++++ crates/vecdb/.gitignore | 5 + crates/vecdb/Cargo.toml | 69 + crates/vecdb/README.md | 321 + crates/vecdb/examples/bench.rs | 639 ++ crates/vecdb/examples/io_vs_mmap.rs | 284 + crates/vecdb/examples/iter_vs_mmap.rs | 89 + crates/vecdb/examples/pco_bench.rs | 104 + crates/vecdb/examples/pcodec.rs | 172 + crates/vecdb/examples/signal_test.rs | 17 + crates/vecdb/examples/zerocopy.rs | 839 +++ crates/vecdb/src/base/change/cursor.rs | 60 + crates/vecdb/src/base/change/data.rs | 11 + crates/vecdb/src/base/change/mod.rs | 5 + crates/vecdb/src/base/format/bytes.rs | 32 + crates/vecdb/src/base/format/mod.rs | 55 + crates/vecdb/src/base/header/inner.rs | 118 + crates/vecdb/src/base/header/mod.rs | 85 + crates/vecdb/src/base/mod.rs | 20 + crates/vecdb/src/base/options/from.rs | 11 + crates/vecdb/src/base/options/mod.rs | 34 + crates/vecdb/src/base/read_only.rs | 57 + crates/vecdb/src/base/read_write.rs | 241 + crates/vecdb/src/base/rollback.rs | 149 + crates/vecdb/src/base/shared_len/default.rs | 7 + crates/vecdb/src/base/shared_len/mod.rs | 32 + crates/vecdb/src/base/with_prev/default.rs | 10 + crates/vecdb/src/base/with_prev/mod.rs | 96 + crates/vecdb/src/bytes/array.rs | 32 + crates/vecdb/src/bytes/mod.rs | 40 + crates/vecdb/src/bytes/numeric.rs | 34 + crates/vecdb/src/cursor.rs | 158 + crates/vecdb/src/error/mod.rs | 127 + crates/vecdb/src/error/zerocopy.rs | 13 + crates/vecdb/src/exit/guard.rs | 31 + crates/vecdb/src/exit/mod.rs | 113 + crates/vecdb/src/iterators/mod.rs | 3 + crates/vecdb/src/iterators/writer.rs | 32 + crates/vecdb/src/lib.rs | 46 + crates/vecdb/src/ops/checked_sub.rs | 19 + crates/vecdb/src/ops/mod.rs | 7 + crates/vecdb/src/ops/saturating_add.rs | 19 + crates/vecdb/src/stamp/bytes.rs | 17 + crates/vecdb/src/stamp/conversions.rs | 13 + crates/vecdb/src/stamp/mod.rs | 15 + crates/vecdb/src/traits/any.rs | 59 + crates/vecdb/src/traits/any_exportable.rs | 9 + crates/vecdb/src/traits/any_readable.rs | 24 + crates/vecdb/src/traits/any_serializable.rs | 107 + crates/vecdb/src/traits/any_stored.rs | 87 + crates/vecdb/src/traits/any_with_schema.rs | 20 + crates/vecdb/src/traits/any_with_writer.rs | 26 + crates/vecdb/src/traits/formattable.rs | 96 + crates/vecdb/src/traits/importable.rs | 21 + crates/vecdb/src/traits/index.rs | 57 + crates/vecdb/src/traits/mod.rs | 43 + crates/vecdb/src/traits/printable.rs | 99 + crates/vecdb/src/traits/read_only_clone.rs | 50 + crates/vecdb/src/traits/readable.rs | 490 ++ crates/vecdb/src/traits/readable_option.rs | 28 + crates/vecdb/src/traits/storage_mode.rs | 26 + crates/vecdb/src/traits/stored.rs | 20 + crates/vecdb/src/traits/typed.rs | 16 + crates/vecdb/src/traits/value.rs | 13 + crates/vecdb/src/traits/value_strategy.rs | 48 + crates/vecdb/src/traits/writable.rs | 222 + crates/vecdb/src/variants/cached/any_vec.rs | 40 + crates/vecdb/src/variants/cached/budget.rs | 28 + crates/vecdb/src/variants/cached/clone.rs | 15 + crates/vecdb/src/variants/cached/cloneable.rs | 52 + crates/vecdb/src/variants/cached/mod.rs | 142 + .../src/variants/cached/read_only_clone.rs | 17 + crates/vecdb/src/variants/cached/readable.rs | 101 + crates/vecdb/src/variants/cached/typed.rs | 8 + .../src/variants/compressed/inner/mod.rs | 11 + .../variants/compressed/inner/page/bytes.rs | 34 + .../src/variants/compressed/inner/page/mod.rs | 54 + .../src/variants/compressed/inner/pages.rs | 121 + .../compressed/inner/read_only/any_vec.rs | 47 + .../compressed/inner/read_only/mod.rs | 106 + .../compressed/inner/read_only/readable.rs | 73 + .../compressed/inner/read_only/typed.rs | 13 + .../inner/read_write/any_stored_vec.rs | 214 + .../compressed/inner/read_write/any_vec.rs | 45 + .../compressed/inner/read_write/mod.rs | 313 + .../compressed/inner/read_write/readable.rs | 104 + .../compressed/inner/read_write/rollback.rs | 45 + .../compressed/inner/read_write/typed.rs | 13 + .../compressed/inner/read_write/writable.rs | 69 + .../src/variants/compressed/inner/strategy.rs | 128 + .../vecdb/src/variants/compressed/lz4/mod.rs | 33 + .../src/variants/compressed/lz4/strategy.rs | 32 + .../src/variants/compressed/lz4/value.rs | 11 + crates/vecdb/src/variants/compressed/mod.rs | 18 + .../vecdb/src/variants/compressed/pco/mod.rs | 36 + .../src/variants/compressed/pco/strategy.rs | 81 + .../src/variants/compressed/pco/trait.rs | 13 + .../variants/compressed/pco/value/as_inner.rs | 33 + .../compressed/pco/value/as_inner_mut.rs | 35 + .../compressed/pco/value/from_inner.rs | 30 + .../src/variants/compressed/pco/value/mod.rs | 28 + .../src/variants/compressed/sources/io.rs | 217 + .../src/variants/compressed/sources/mmap.rs | 138 + .../src/variants/compressed/sources/mod.rs | 5 + .../vecdb/src/variants/compressed/zstd/mod.rs | 32 + .../src/variants/compressed/zstd/strategy.rs | 37 + .../src/variants/compressed/zstd/value.rs | 11 + .../src/variants/eager/any_stored_vec.rs | 78 + crates/vecdb/src/variants/eager/any_vec.rs | 43 + .../src/variants/eager/compute/aggregates.rs | 470 ++ .../src/variants/eager/compute/arithmetic.rs | 176 + .../src/variants/eager/compute/cumulative.rs | 242 + .../src/variants/eager/compute/lookback.rs | 374 + .../vecdb/src/variants/eager/compute/mod.rs | 6 + .../src/variants/eager/compute/statistics.rs | 1183 ++++ .../src/variants/eager/compute/transforms.rs | 372 + crates/vecdb/src/variants/eager/importable.rs | 23 + crates/vecdb/src/variants/eager/mod.rs | 90 + crates/vecdb/src/variants/eager/readable.rs | 45 + .../src/variants/eager/readable_cloneable.rs | 13 + crates/vecdb/src/variants/eager/stored.rs | 15 + crates/vecdb/src/variants/eager/typed.rs | 11 + crates/vecdb/src/variants/eager/writable.rs | 58 + crates/vecdb/src/variants/lazy/agg/any_vec.rs | 44 + crates/vecdb/src/variants/lazy/agg/clone.rs | 25 + crates/vecdb/src/variants/lazy/agg/fold.rs | 46 + crates/vecdb/src/variants/lazy/agg/mod.rs | 61 + .../vecdb/src/variants/lazy/agg/readable.rs | 73 + crates/vecdb/src/variants/lazy/agg/sparse.rs | 65 + crates/vecdb/src/variants/lazy/agg/typed.rs | 16 + .../vecdb/src/variants/lazy/delta/any_vec.rs | 42 + crates/vecdb/src/variants/lazy/delta/avg.rs | 35 + .../vecdb/src/variants/lazy/delta/change.rs | 16 + crates/vecdb/src/variants/lazy/delta/clone.rs | 22 + crates/vecdb/src/variants/lazy/delta/mod.rs | 114 + crates/vecdb/src/variants/lazy/delta/op.rs | 30 + crates/vecdb/src/variants/lazy/delta/rate.rs | 22 + .../vecdb/src/variants/lazy/delta/readable.rs | 145 + crates/vecdb/src/variants/lazy/delta/sub.rs | 32 + crates/vecdb/src/variants/lazy/delta/typed.rs | 14 + .../vecdb/src/variants/lazy/from1/any_vec.rs | 42 + crates/vecdb/src/variants/lazy/from1/mod.rs | 81 + .../variants/lazy/from1/read_only_clone.rs | 17 + .../vecdb/src/variants/lazy/from1/readable.rs | 80 + .../variants/lazy/from1/transform/halve.rs | 13 + .../variants/lazy/from1/transform/ident.rs | 11 + .../src/variants/lazy/from1/transform/mod.rs | 13 + .../variants/lazy/from1/transform/negate.rs | 13 + crates/vecdb/src/variants/lazy/from1/typed.rs | 14 + .../vecdb/src/variants/lazy/from2/any_vec.rs | 54 + crates/vecdb/src/variants/lazy/from2/mod.rs | 95 + .../vecdb/src/variants/lazy/from2/readable.rs | 92 + .../variants/lazy/from2/transform/divide.rs | 13 + .../variants/lazy/from2/transform/minus.rs | 13 + .../src/variants/lazy/from2/transform/mod.rs | 15 + .../src/variants/lazy/from2/transform/plus.rs | 13 + .../variants/lazy/from2/transform/times.rs | 13 + crates/vecdb/src/variants/lazy/from2/typed.rs | 16 + .../vecdb/src/variants/lazy/from3/any_vec.rs | 61 + crates/vecdb/src/variants/lazy/from3/mod.rs | 85 + .../vecdb/src/variants/lazy/from3/readable.rs | 101 + crates/vecdb/src/variants/lazy/from3/typed.rs | 19 + crates/vecdb/src/variants/lazy/mod.rs | 11 + crates/vecdb/src/variants/macros.rs | 332 + crates/vecdb/src/variants/mod.rs | 14 + crates/vecdb/src/variants/raw/bytes/mod.rs | 45 + .../src/variants/raw/bytes/strategy/mod.rs | 10 + .../src/variants/raw/bytes/strategy/raw.rs | 17 + .../src/variants/raw/bytes/strategy/value.rs | 22 + crates/vecdb/src/variants/raw/bytes/value.rs | 11 + crates/vecdb/src/variants/raw/inner/mod.rs | 7 + .../variants/raw/inner/read_only/any_vec.rs | 45 + .../src/variants/raw/inner/read_only/mod.rs | 116 + .../variants/raw/inner/read_only/readable.rs | 86 + .../src/variants/raw/inner/read_only/typed.rs | 13 + .../raw/inner/read_write/any_stored_vec.rs | 168 + .../variants/raw/inner/read_write/any_vec.rs | 49 + .../variants/raw/inner/read_write/change.rs | 9 + .../src/variants/raw/inner/read_write/mod.rs | 594 ++ .../variants/raw/inner/read_write/readable.rs | 155 + .../variants/raw/inner/read_write/rollback.rs | 123 + .../variants/raw/inner/read_write/typed.rs | 13 + .../variants/raw/inner/read_write/writable.rs | 80 + .../vecdb/src/variants/raw/inner/strategy.rs | 16 + crates/vecdb/src/variants/raw/mod.rs | 12 + crates/vecdb/src/variants/raw/sources/io.rs | 157 + crates/vecdb/src/variants/raw/sources/mmap.rs | 98 + crates/vecdb/src/variants/raw/sources/mod.rs | 7 + .../vecdb/src/variants/raw/sources/reader.rs | 203 + crates/vecdb/src/variants/raw/zerocopy/mod.rs | 112 + .../src/variants/raw/zerocopy/strategy/mod.rs | 10 + .../src/variants/raw/zerocopy/strategy/raw.rs | 10 + .../variants/raw/zerocopy/strategy/value.rs | 24 + .../vecdb/src/variants/raw/zerocopy/value.rs | 13 + crates/vecdb/src/version/add.rs | 10 + crates/vecdb/src/version/bytes.rs | 17 + crates/vecdb/src/version/conversions.rs | 26 + crates/vecdb/src/version/display.rs | 9 + crates/vecdb/src/version/mod.rs | 39 + crates/vecdb/src/version/sum.rs | 9 + crates/vecdb/src/version/try_from_path.rs | 14 + crates/vecdb/tests/basic.rs | 261 + crates/vecdb/tests/basic2.rs | 592 ++ crates/vecdb/tests/comprehensive.rs | 656 ++ crates/vecdb/tests/compute_functions.rs | 1477 ++++ crates/vecdb/tests/concurrent_rw.rs | 1016 +++ crates/vecdb/tests/consistency.rs | 180 + crates/vecdb/tests/cursor.rs | 84 + crates/vecdb/tests/cursor_chunk_size.rs | 29 + crates/vecdb/tests/integrity.rs | 518 ++ crates/vecdb/tests/iterators.rs | 1795 +++++ crates/vecdb/tests/non_copy.rs | 42 + crates/vecdb/tests/raw_last_page.rs | 1374 ++++ crates/vecdb/tests/raw_reader_cursor.rs | 70 + crates/vecdb/tests/rollback.rs | 1856 +++++ crates/vecdb/tests/rollback_truncation.rs | 179 + crates/vecdb/tests/scaling_bench.rs | 256 + crates/vecdb/tests/write_visibility.rs | 272 + crates/vecdb_derive/Cargo.toml | 22 + crates/vecdb_derive/README.md | 133 + crates/vecdb_derive/src/lib.rs | 251 + crates/vecdb_derive/tests/bytes.rs | 76 + crates/vecdb_derive/tests/bytes_generics.rs | 122 + crates/vecdb_derive/tests/pco.rs | 65 + crates/vecdb_derive/tests/pco_generics.rs | 139 + 732 files changed, 97745 insertions(+), 1689 deletions(-) create mode 100644 crates/brk_store/tests/owned_ingest.rs create mode 100644 crates/fjall/.config/nextest.toml create mode 100644 crates/fjall/.gitignore create mode 100644 crates/fjall/.rustfmt.toml create mode 100644 crates/fjall/.vscode/settings.json create mode 100644 crates/fjall/CHANGELOG.md create mode 100644 crates/fjall/CONTRIBUTING.md create mode 100644 crates/fjall/Cargo.toml create mode 100644 crates/fjall/LICENSE-APACHE create mode 100644 crates/fjall/LICENSE-MIT create mode 100644 crates/fjall/README.md create mode 100644 crates/fjall/commit.nu create mode 100644 crates/fjall/compile_examples.mjs create mode 100644 crates/fjall/kawaii.png create mode 100644 crates/fjall/logo.png create mode 100644 crates/fjall/renovate.json create mode 100644 crates/fjall/src/batch/item.rs create mode 100644 crates/fjall/src/batch/mod.rs create mode 100644 crates/fjall/src/builder.rs create mode 100644 crates/fjall/src/compaction/mod.rs create mode 100644 crates/fjall/src/compaction/worker.rs create mode 100644 crates/fjall/src/db.rs create mode 100644 crates/fjall/src/db_config.rs create mode 100644 crates/fjall/src/db_test.rs create mode 100644 crates/fjall/src/drop.rs create mode 100644 crates/fjall/src/error.rs create mode 100644 crates/fjall/src/file.rs create mode 100644 crates/fjall/src/flush/manager.rs create mode 100644 crates/fjall/src/flush/mod.rs create mode 100644 crates/fjall/src/flush/task.rs create mode 100644 crates/fjall/src/flush/worker.rs create mode 100644 crates/fjall/src/guard.rs create mode 100644 crates/fjall/src/ingestion.rs create mode 100644 crates/fjall/src/iter.rs create mode 100644 crates/fjall/src/journal/batch_reader.rs create mode 100644 crates/fjall/src/journal/entry.rs create mode 100644 crates/fjall/src/journal/error.rs create mode 100644 crates/fjall/src/journal/manager.rs create mode 100644 crates/fjall/src/journal/mod.rs create mode 100644 crates/fjall/src/journal/reader.rs create mode 100644 crates/fjall/src/journal/recovery.rs create mode 100644 crates/fjall/src/journal/test.rs create mode 100644 crates/fjall/src/journal/writer.rs create mode 100644 crates/fjall/src/keyspace/config/block_size.rs create mode 100644 crates/fjall/src/keyspace/config/compression.rs create mode 100644 crates/fjall/src/keyspace/config/filter.rs create mode 100644 crates/fjall/src/keyspace/config/hash_ratio.rs create mode 100644 crates/fjall/src/keyspace/config/mod.rs create mode 100644 crates/fjall/src/keyspace/config/pinning.rs create mode 100644 crates/fjall/src/keyspace/config/restart_interval.rs create mode 100644 crates/fjall/src/keyspace/mod.rs create mode 100644 crates/fjall/src/keyspace/name.rs create mode 100644 crates/fjall/src/keyspace/options.rs create mode 100644 crates/fjall/src/keyspace/test.rs create mode 100644 crates/fjall/src/keyspace/write_delay.rs create mode 100644 crates/fjall/src/lib.rs create mode 100644 crates/fjall/src/locked_file.rs create mode 100644 crates/fjall/src/meta_keyspace.rs create mode 100644 crates/fjall/src/path.rs create mode 100644 crates/fjall/src/poison.rs create mode 100644 crates/fjall/src/readable.rs create mode 100644 crates/fjall/src/recovery.rs create mode 100644 crates/fjall/src/snapshot.rs create mode 100644 crates/fjall/src/snapshot_nonce.rs create mode 100644 crates/fjall/src/snapshot_tracker.rs create mode 100644 crates/fjall/src/stats.rs create mode 100644 crates/fjall/src/supervisor.rs create mode 100644 crates/fjall/src/tx/mod.rs create mode 100644 crates/fjall/src/tx/optimistic/conflict_manager.rs create mode 100644 crates/fjall/src/tx/optimistic/keyspace.rs create mode 100644 crates/fjall/src/tx/optimistic/mod.rs create mode 100644 crates/fjall/src/tx/optimistic/oracle.rs create mode 100644 crates/fjall/src/tx/optimistic/write_tx.rs create mode 100644 crates/fjall/src/tx/single_writer/keyspace.rs create mode 100644 crates/fjall/src/tx/single_writer/mod.rs create mode 100644 crates/fjall/src/tx/single_writer/write_tx.rs create mode 100644 crates/fjall/src/tx/write_tx.rs create mode 100644 crates/fjall/src/version.rs create mode 100644 crates/fjall/src/worker_pool.rs create mode 100644 crates/fjall/src/write_buffer_manager.rs create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/a/config create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/a/levels create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/a/segments/0 create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/a/version create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/b/config create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/b/levels create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/b/segments/1 create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/b/version create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/c/config create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/c/levels create mode 100644 crates/fjall/test_fixture/v1_keyspace/partitions/c/version create mode 100644 crates/fjall/test_fixture/v1_keyspace/version create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/config create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/levels create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/segments/0 create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/version create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/config create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/levels create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/segments/1 create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/version create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/config create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/levels create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/version create mode 100644 crates/fjall/test_fixture/v1_keyspace_corrupt_journal/version create mode 100644 crates/fjall/test_fixture/v2_keyspace/journals/2 create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default1/config create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default1/levels create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default1/manifest create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default1/segments/0 create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/blobs/.vlog create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/blobs/segments/0 create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/blobs/vlog_manifest create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/config create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/levels create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/manifest create mode 100644 crates/fjall/test_fixture/v2_keyspace/partitions/default2/segments/0 create mode 100644 crates/fjall/test_fixture/v2_keyspace/version create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/journals/2 create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/config create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/levels create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/manifest create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/segments/0 create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/.vlog create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/segments/0 create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/blobs/vlog_manifest create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/config create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/levels create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/manifest create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/segments/0 create mode 100644 crates/fjall/test_fixture/v2_keyspace_corrupt_journal/version create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/0.sealed create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/1.sealed create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/2 create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/3 create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/partitions/default/config create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/partitions/default/levels create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/partitions/default/manifest create mode 100644 crates/fjall/test_fixture/v2_sealed_journal_shenanigans/version create mode 100644 crates/fjall/tests/batch.rs create mode 100644 crates/fjall/tests/batch_recovery.rs create mode 100644 crates/fjall/tests/blob_kv_simple.rs create mode 100644 crates/fjall/tests/clear_dirty_read.rs create mode 100644 crates/fjall/tests/compaction_filter.rs create mode 100644 crates/fjall/tests/db_lock.rs create mode 100644 crates/fjall/tests/db_open.rs create mode 100644 crates/fjall/tests/fifo_dirty_read.rs create mode 100644 crates/fjall/tests/format_v4.rs create mode 100644 crates/fjall/tests/ingest_recovery.rs create mode 100644 crates/fjall/tests/journal_large_value.rs create mode 100644 crates/fjall/tests/keyspace_clear.rs create mode 100644 crates/fjall/tests/keyspace_iter_lifetime.rs create mode 100644 crates/fjall/tests/keyspace_recover.rs create mode 100644 crates/fjall/tests/keyspace_snapshot.rs create mode 100644 crates/fjall/tests/keyspace_v1_load_fixture.rs create mode 100644 crates/fjall/tests/keyspace_v2_load_fixture.rs create mode 100644 crates/fjall/tests/memtable_recover.rs create mode 100644 crates/fjall/tests/prefix_complex.rs create mode 100644 crates/fjall/tests/recover_from_different_folder.rs create mode 100644 crates/fjall/tests/recovery_journal_mac.rs create mode 100644 crates/fjall/tests/recovery_keyspaces_mac.rs create mode 100644 crates/fjall/tests/seqno_recovery.rs create mode 100644 crates/fjall/tests/standard_keyspace.rs create mode 100644 crates/fjall/tests/tx_ryow.rs create mode 100644 crates/fjall/tests/tx_ryow_snapshot.rs create mode 100644 crates/fjall/tests/write_buffer_size.rs create mode 100644 crates/fjall/tests/write_during_read.rs create mode 100644 crates/fjall/tests/write_tx.rs create mode 100644 crates/lsm-tree/.config/nextest.toml create mode 100644 crates/lsm-tree/.gitignore create mode 100644 crates/lsm-tree/.rustfmt.toml create mode 100644 crates/lsm-tree/CONTRIBUTING.md create mode 100644 crates/lsm-tree/Cargo.toml create mode 100644 crates/lsm-tree/LICENSE-APACHE create mode 100644 crates/lsm-tree/LICENSE-MIT create mode 100644 crates/lsm-tree/README.md create mode 100644 crates/lsm-tree/UNSAFE.md create mode 100644 crates/lsm-tree/benches/block.rs create mode 100644 crates/lsm-tree/benches/bloom.rs create mode 100644 crates/lsm-tree/benches/fd_table.rs create mode 100644 crates/lsm-tree/benches/level_manifest.rs create mode 100644 crates/lsm-tree/benches/memtable.rs create mode 100644 crates/lsm-tree/benches/merge.rs create mode 100644 crates/lsm-tree/benches/tree.rs create mode 100644 crates/lsm-tree/logo.png create mode 100644 crates/lsm-tree/renovate.json create mode 100644 crates/lsm-tree/src/abstract_tree.rs create mode 100644 crates/lsm-tree/src/any_tree.rs create mode 100644 crates/lsm-tree/src/blob_tree/gc.rs create mode 100644 crates/lsm-tree/src/blob_tree/handle.rs create mode 100644 crates/lsm-tree/src/blob_tree/ingest.rs create mode 100644 crates/lsm-tree/src/blob_tree/mod.rs create mode 100644 crates/lsm-tree/src/cache.rs create mode 100644 crates/lsm-tree/src/checksum.rs create mode 100644 crates/lsm-tree/src/coding.rs create mode 100644 crates/lsm-tree/src/compaction/drop_range.rs create mode 100644 crates/lsm-tree/src/compaction/fifo.rs create mode 100644 crates/lsm-tree/src/compaction/filter.rs create mode 100644 crates/lsm-tree/src/compaction/flavour.rs create mode 100644 crates/lsm-tree/src/compaction/leveled/mod.rs create mode 100644 crates/lsm-tree/src/compaction/leveled/test.rs create mode 100644 crates/lsm-tree/src/compaction/maintenance.rs create mode 100644 crates/lsm-tree/src/compaction/major.rs create mode 100644 crates/lsm-tree/src/compaction/mod.rs create mode 100644 crates/lsm-tree/src/compaction/movedown.rs create mode 100644 crates/lsm-tree/src/compaction/pulldown.rs create mode 100644 crates/lsm-tree/src/compaction/state/hidden_set.rs create mode 100644 crates/lsm-tree/src/compaction/state/mod.rs create mode 100644 crates/lsm-tree/src/compaction/stream.rs create mode 100644 crates/lsm-tree/src/compaction/tiered.rs create mode 100644 crates/lsm-tree/src/compaction/worker.rs create mode 100644 crates/lsm-tree/src/compression.rs create mode 100644 crates/lsm-tree/src/config/block_size.rs create mode 100644 crates/lsm-tree/src/config/compression.rs create mode 100644 crates/lsm-tree/src/config/filter.rs create mode 100644 crates/lsm-tree/src/config/hash_ratio.rs create mode 100644 crates/lsm-tree/src/config/mod.rs create mode 100644 crates/lsm-tree/src/config/pinning.rs create mode 100644 crates/lsm-tree/src/config/restart_interval.rs create mode 100644 crates/lsm-tree/src/descriptor_table.rs create mode 100644 crates/lsm-tree/src/double_ended_peekable.rs create mode 100644 crates/lsm-tree/src/error.rs create mode 100644 crates/lsm-tree/src/file.rs create mode 100644 crates/lsm-tree/src/file_accessor.rs create mode 100644 crates/lsm-tree/src/format_version.rs create mode 100644 crates/lsm-tree/src/hash.rs create mode 100644 crates/lsm-tree/src/ingestion.rs create mode 100644 crates/lsm-tree/src/iter_guard.rs create mode 100644 crates/lsm-tree/src/key.rs create mode 100644 crates/lsm-tree/src/key_range.rs create mode 100644 crates/lsm-tree/src/lib.rs create mode 100644 crates/lsm-tree/src/manifest.rs create mode 100644 crates/lsm-tree/src/memtable/mod.rs create mode 100644 crates/lsm-tree/src/merge.rs create mode 100644 crates/lsm-tree/src/metrics.rs create mode 100644 crates/lsm-tree/src/mvcc_stream.rs create mode 100644 crates/lsm-tree/src/path.rs create mode 100644 crates/lsm-tree/src/range.rs create mode 100644 crates/lsm-tree/src/run_reader.rs create mode 100644 crates/lsm-tree/src/run_scanner.rs create mode 100644 crates/lsm-tree/src/seqno.rs create mode 100644 crates/lsm-tree/src/slice/mod.rs create mode 100644 crates/lsm-tree/src/slice/slice_bytes/mod.rs create mode 100644 crates/lsm-tree/src/slice/slice_default/mod.rs create mode 100644 crates/lsm-tree/src/slice_windows.rs create mode 100644 crates/lsm-tree/src/stop_signal.rs create mode 100644 crates/lsm-tree/src/table/block/binary_index/builder.rs create mode 100644 crates/lsm-tree/src/table/block/binary_index/mod.rs create mode 100644 crates/lsm-tree/src/table/block/binary_index/reader.rs create mode 100644 crates/lsm-tree/src/table/block/decoder.rs create mode 100644 crates/lsm-tree/src/table/block/encoder.rs create mode 100644 crates/lsm-tree/src/table/block/hash_index/builder.rs create mode 100644 crates/lsm-tree/src/table/block/hash_index/mod.rs create mode 100644 crates/lsm-tree/src/table/block/hash_index/reader.rs create mode 100644 crates/lsm-tree/src/table/block/header.rs create mode 100644 crates/lsm-tree/src/table/block/mod.rs create mode 100644 crates/lsm-tree/src/table/block/offset.rs create mode 100644 crates/lsm-tree/src/table/block/trailer.rs create mode 100644 crates/lsm-tree/src/table/block/type.rs create mode 100644 crates/lsm-tree/src/table/block_index/full.rs create mode 100644 crates/lsm-tree/src/table/block_index/iter.rs create mode 100644 crates/lsm-tree/src/table/block_index/mod.rs create mode 100644 crates/lsm-tree/src/table/block_index/two_level.rs create mode 100644 crates/lsm-tree/src/table/block_index/volatile.rs create mode 100644 crates/lsm-tree/src/table/data_block/iter.rs create mode 100644 crates/lsm-tree/src/table/data_block/iter_test.rs create mode 100644 crates/lsm-tree/src/table/data_block/mod.rs create mode 100644 crates/lsm-tree/src/table/filter/bit_array/builder.rs create mode 100644 crates/lsm-tree/src/table/filter/bit_array/mod.rs create mode 100644 crates/lsm-tree/src/table/filter/bit_array/reader.rs create mode 100644 crates/lsm-tree/src/table/filter/block.rs create mode 100644 crates/lsm-tree/src/table/filter/blocked_bloom/builder._rs create mode 100644 crates/lsm-tree/src/table/filter/blocked_bloom/mod._rs create mode 100644 crates/lsm-tree/src/table/filter/mod.rs create mode 100644 crates/lsm-tree/src/table/filter/standard_bloom/builder.rs create mode 100644 crates/lsm-tree/src/table/filter/standard_bloom/mod.rs create mode 100644 crates/lsm-tree/src/table/id.rs create mode 100644 crates/lsm-tree/src/table/index_block/block_handle.rs create mode 100644 crates/lsm-tree/src/table/index_block/iter.rs create mode 100644 crates/lsm-tree/src/table/index_block/mod.rs create mode 100644 crates/lsm-tree/src/table/inner.rs create mode 100644 crates/lsm-tree/src/table/iter.rs create mode 100644 crates/lsm-tree/src/table/meta.rs create mode 100644 crates/lsm-tree/src/table/mod.rs create mode 100644 crates/lsm-tree/src/table/multi_writer.rs create mode 100644 crates/lsm-tree/src/table/regions.rs create mode 100644 crates/lsm-tree/src/table/scanner.rs create mode 100644 crates/lsm-tree/src/table/tests.rs create mode 100644 crates/lsm-tree/src/table/util.rs create mode 100644 crates/lsm-tree/src/table/writer/filter/full.rs create mode 100644 crates/lsm-tree/src/table/writer/filter/mod.rs create mode 100644 crates/lsm-tree/src/table/writer/filter/partitioned.rs create mode 100644 crates/lsm-tree/src/table/writer/index/full.rs create mode 100644 crates/lsm-tree/src/table/writer/index/mod.rs create mode 100644 crates/lsm-tree/src/table/writer/index/partitioned.rs create mode 100644 crates/lsm-tree/src/table/writer/meta.rs create mode 100644 crates/lsm-tree/src/table/writer/mod.rs create mode 100644 crates/lsm-tree/src/time.rs create mode 100644 crates/lsm-tree/src/tree/ingest.rs create mode 100644 crates/lsm-tree/src/tree/inner.rs create mode 100644 crates/lsm-tree/src/tree/mod.rs create mode 100644 crates/lsm-tree/src/tree/sealed.rs create mode 100644 crates/lsm-tree/src/util.rs create mode 100644 crates/lsm-tree/src/value.rs create mode 100644 crates/lsm-tree/src/value_type.rs create mode 100644 crates/lsm-tree/src/version/blob_file_list.rs create mode 100644 crates/lsm-tree/src/version/mod.rs create mode 100644 crates/lsm-tree/src/version/optimize.rs create mode 100644 crates/lsm-tree/src/version/persist.rs create mode 100644 crates/lsm-tree/src/version/recovery.rs create mode 100644 crates/lsm-tree/src/version/run.rs create mode 100644 crates/lsm-tree/src/version/super_version.rs create mode 100644 crates/lsm-tree/src/vlog/accessor.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/merge.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/meta.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/mod.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/multi_writer.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/reader.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/scanner.rs create mode 100644 crates/lsm-tree/src/vlog/blob_file/writer.rs create mode 100644 crates/lsm-tree/src/vlog/handle.rs create mode 100644 crates/lsm-tree/src/vlog/mod.rs create mode 100644 crates/lsm-tree/test_fixture/v1_tree/config create mode 100644 crates/lsm-tree/test_fixture/v1_tree/levels create mode 100644 crates/lsm-tree/test_fixture/v1_tree/segments/0 create mode 100644 crates/lsm-tree/test_fixture/v1_tree/version create mode 100644 crates/lsm-tree/test_fixture/v1_tree_corrupt/config create mode 100644 crates/lsm-tree/test_fixture/v1_tree_corrupt/levels create mode 100644 crates/lsm-tree/test_fixture/v1_tree_corrupt/segments/0 create mode 100644 crates/lsm-tree/test_fixture/v1_tree_corrupt/version create mode 100644 crates/lsm-tree/tests/blob_compression.rs create mode 100644 crates/lsm-tree/tests/blob_empty_flush.rs create mode 100644 crates/lsm-tree/tests/blob_fifo_limit.rs create mode 100644 crates/lsm-tree/tests/blob_file_full_file_checksum.rs create mode 100644 crates/lsm-tree/tests/blob_flush_gc_stats.rs create mode 100644 crates/lsm-tree/tests/blob_gc_watermark.rs create mode 100644 crates/lsm-tree/tests/blob_ingest.rs create mode 100644 crates/lsm-tree/tests/blob_ingest_relink.rs create mode 100644 crates/lsm-tree/tests/blob_major_compact_drop_dead_files.rs create mode 100644 crates/lsm-tree/tests/blob_major_compact_gc_stats.rs create mode 100644 crates/lsm-tree/tests/blob_major_compact_relink.rs create mode 100644 crates/lsm-tree/tests/blob_major_compact_relocation.rs create mode 100644 crates/lsm-tree/tests/blob_major_compact_relocation_recovery.rs create mode 100644 crates/lsm-tree/tests/blob_nuke_gc_stats.rs create mode 100644 crates/lsm-tree/tests/blob_pinned_fd.rs create mode 100644 crates/lsm-tree/tests/blob_recover_gc_stats.rs create mode 100644 crates/lsm-tree/tests/blob_recovery.rs create mode 100644 crates/lsm-tree/tests/blob_register_table_rotation.rs create mode 100644 crates/lsm-tree/tests/blob_sep_threshold.rs create mode 100644 crates/lsm-tree/tests/blob_simple.rs create mode 100644 crates/lsm-tree/tests/blob_tree_guarded_size.rs create mode 100644 crates/lsm-tree/tests/blob_tree_reload_blob.rs create mode 100644 crates/lsm-tree/tests/compaction_filter.rs create mode 100644 crates/lsm-tree/tests/compaction_filter_gc_stats_change.rs create mode 100644 crates/lsm-tree/tests/compaction_filter_gc_stats_destroy.rs create mode 100644 crates/lsm-tree/tests/compaction_filter_gc_stats_remove.rs create mode 100644 crates/lsm-tree/tests/compaction_filter_gc_stats_remove_weak.rs create mode 100644 crates/lsm-tree/tests/compaction_filter_snapshot.rs create mode 100644 crates/lsm-tree/tests/compaction_filter_ttl.rs create mode 100644 crates/lsm-tree/tests/compaction_readers_grouping.rs create mode 100644 crates/lsm-tree/tests/dirty_snapshot.rs create mode 100644 crates/lsm-tree/tests/expect_point_read_hits.rs create mode 100644 crates/lsm-tree/tests/fifo_ttl.rs create mode 100644 crates/lsm-tree/tests/guard_if.rs create mode 100644 crates/lsm-tree/tests/ingest_dirty_snapshot.rs create mode 100644 crates/lsm-tree/tests/ingestion_api.rs create mode 100644 crates/lsm-tree/tests/ingestion_invariants.rs create mode 100644 crates/lsm-tree/tests/ingestion_seqno.rs create mode 100644 crates/lsm-tree/tests/latest_snapshot.rs create mode 100644 crates/lsm-tree/tests/leveled_trivial_move.rs create mode 100644 crates/lsm-tree/tests/memtable_id.rs create mode 100644 crates/lsm-tree/tests/model_1.rs create mode 100644 crates/lsm-tree/tests/model_2.rs create mode 100644 crates/lsm-tree/tests/model_3.rs create mode 100644 crates/lsm-tree/tests/model_4.rs create mode 100644 crates/lsm-tree/tests/model_5.rs create mode 100644 crates/lsm-tree/tests/model_6.rs create mode 100644 crates/lsm-tree/tests/multi_trees.rs create mode 100644 crates/lsm-tree/tests/mvcc_slab.rs create mode 100644 crates/lsm-tree/tests/recover_cleanup_orphans.rs create mode 100644 crates/lsm-tree/tests/recover_from_different_folder.rs create mode 100644 crates/lsm-tree/tests/recovery_mac_ds_store.rs create mode 100644 crates/lsm-tree/tests/recovery_mac_underscore.rs create mode 100644 crates/lsm-tree/tests/regression_286.rs create mode 100644 crates/lsm-tree/tests/snapshot_compact.rs create mode 100644 crates/lsm-tree/tests/snapshot_len.rs create mode 100644 crates/lsm-tree/tests/snapshot_point_read.rs create mode 100644 crates/lsm-tree/tests/snapshot_zombie.rs create mode 100644 crates/lsm-tree/tests/static_iterators.rs create mode 100644 crates/lsm-tree/tests/table_full_file_checksum.rs create mode 100644 crates/lsm-tree/tests/table_point_reads.rs create mode 100644 crates/lsm-tree/tests/table_range.rs create mode 100644 crates/lsm-tree/tests/table_range_oob.rs create mode 100644 crates/lsm-tree/tests/table_remove_weak.rs create mode 100644 crates/lsm-tree/tests/table_weak_tombstones.rs create mode 100644 crates/lsm-tree/tests/tree_approx_len.rs create mode 100644 crates/lsm-tree/tests/tree_bulk_ingest.rs create mode 100644 crates/lsm-tree/tests/tree_clear.rs create mode 100644 crates/lsm-tree/tests/tree_clear_snapshot.rs create mode 100644 crates/lsm-tree/tests/tree_count.rs create mode 100644 crates/lsm-tree/tests/tree_delete_loop.rs create mode 100644 crates/lsm-tree/tests/tree_different_block_size.rs create mode 100644 crates/lsm-tree/tests/tree_disjoint_iter.rs create mode 100644 crates/lsm-tree/tests/tree_disjoint_point_read.rs create mode 100644 crates/lsm-tree/tests/tree_disjoint_prefix.rs create mode 100644 crates/lsm-tree/tests/tree_disjoint_range.rs create mode 100644 crates/lsm-tree/tests/tree_drop_range.rs create mode 100644 crates/lsm-tree/tests/tree_filter_hit_rate.rs create mode 100644 crates/lsm-tree/tests/tree_flush_eviction.rs create mode 100644 crates/lsm-tree/tests/tree_guarded_range.rs create mode 100644 crates/lsm-tree/tests/tree_kv.rs create mode 100644 crates/lsm-tree/tests/tree_l0_point_read.rs create mode 100644 crates/lsm-tree/tests/tree_l0_range.rs create mode 100644 crates/lsm-tree/tests/tree_major_compaction.rs create mode 100644 crates/lsm-tree/tests/tree_mvcc_simple.rs create mode 100644 crates/lsm-tree/tests/tree_non_disjoint_point_read.rs create mode 100644 crates/lsm-tree/tests/tree_range.rs create mode 100644 crates/lsm-tree/tests/tree_range_memtable_only.rs create mode 100644 crates/lsm-tree/tests/tree_recover_counter.rs create mode 100644 crates/lsm-tree/tests/tree_recover_large_value.rs create mode 100644 crates/lsm-tree/tests/tree_recovery_versions.rs create mode 100644 crates/lsm-tree/tests/tree_reload.rs create mode 100644 crates/lsm-tree/tests/tree_reload_pwd.rs create mode 100644 crates/lsm-tree/tests/tree_sealed_shadowing.rs create mode 100644 crates/lsm-tree/tests/tree_seqno.rs create mode 100644 crates/lsm-tree/tests/tree_shadowing.rs create mode 100644 crates/lsm-tree/tests/tree_trace.rs create mode 100644 crates/lsm-tree/tests/tree_v1_load_fixture.rs create mode 100644 crates/lsm-tree/tests/tree_weak_delete.rs create mode 100644 crates/lsm-tree/tests/tree_weak_delete_eviction.rs create mode 100644 crates/lsm-tree/tests/tree_weak_delete_queue.rs create mode 100644 crates/lsm-tree/tests/tree_write_read.rs create mode 100644 crates/quickmatch/.gitignore create mode 100644 crates/quickmatch/Cargo.toml create mode 100644 crates/quickmatch/docs/CHANGELOG.md create mode 100644 crates/quickmatch/docs/LICENSE.md create mode 100644 crates/quickmatch/docs/README.md create mode 100644 crates/quickmatch/examples/autocomplete.rs create mode 100644 crates/quickmatch/examples/colon_test.rs create mode 100644 crates/quickmatch/examples/compare.rs create mode 100644 crates/quickmatch/src/config.rs create mode 100644 crates/quickmatch/src/lib.rs create mode 100644 crates/rawdb/Cargo.toml create mode 100644 crates/rawdb/README.md create mode 100644 crates/rawdb/src/disk_usage.rs create mode 100644 crates/rawdb/src/error.rs create mode 100644 crates/rawdb/src/hints.rs create mode 100644 crates/rawdb/src/hole_punch.rs create mode 100644 crates/rawdb/src/layout.rs create mode 100644 crates/rawdb/src/lib.rs create mode 100644 crates/rawdb/src/mmap.rs create mode 100644 crates/rawdb/src/reader.rs create mode 100644 crates/rawdb/src/region.rs create mode 100644 crates/rawdb/src/region_metadata.rs create mode 100644 crates/rawdb/src/region_state.rs create mode 100644 crates/rawdb/src/regions.rs create mode 100644 crates/rawdb/tests/mod.rs create mode 100644 crates/vecdb/.gitignore create mode 100644 crates/vecdb/Cargo.toml create mode 100644 crates/vecdb/README.md create mode 100644 crates/vecdb/examples/bench.rs create mode 100644 crates/vecdb/examples/io_vs_mmap.rs create mode 100644 crates/vecdb/examples/iter_vs_mmap.rs create mode 100644 crates/vecdb/examples/pco_bench.rs create mode 100644 crates/vecdb/examples/pcodec.rs create mode 100644 crates/vecdb/examples/signal_test.rs create mode 100644 crates/vecdb/examples/zerocopy.rs create mode 100644 crates/vecdb/src/base/change/cursor.rs create mode 100644 crates/vecdb/src/base/change/data.rs create mode 100644 crates/vecdb/src/base/change/mod.rs create mode 100644 crates/vecdb/src/base/format/bytes.rs create mode 100644 crates/vecdb/src/base/format/mod.rs create mode 100644 crates/vecdb/src/base/header/inner.rs create mode 100644 crates/vecdb/src/base/header/mod.rs create mode 100644 crates/vecdb/src/base/mod.rs create mode 100644 crates/vecdb/src/base/options/from.rs create mode 100644 crates/vecdb/src/base/options/mod.rs create mode 100644 crates/vecdb/src/base/read_only.rs create mode 100644 crates/vecdb/src/base/read_write.rs create mode 100644 crates/vecdb/src/base/rollback.rs create mode 100644 crates/vecdb/src/base/shared_len/default.rs create mode 100644 crates/vecdb/src/base/shared_len/mod.rs create mode 100644 crates/vecdb/src/base/with_prev/default.rs create mode 100644 crates/vecdb/src/base/with_prev/mod.rs create mode 100644 crates/vecdb/src/bytes/array.rs create mode 100644 crates/vecdb/src/bytes/mod.rs create mode 100644 crates/vecdb/src/bytes/numeric.rs create mode 100644 crates/vecdb/src/cursor.rs create mode 100644 crates/vecdb/src/error/mod.rs create mode 100644 crates/vecdb/src/error/zerocopy.rs create mode 100644 crates/vecdb/src/exit/guard.rs create mode 100644 crates/vecdb/src/exit/mod.rs create mode 100644 crates/vecdb/src/iterators/mod.rs create mode 100644 crates/vecdb/src/iterators/writer.rs create mode 100644 crates/vecdb/src/lib.rs create mode 100644 crates/vecdb/src/ops/checked_sub.rs create mode 100644 crates/vecdb/src/ops/mod.rs create mode 100644 crates/vecdb/src/ops/saturating_add.rs create mode 100644 crates/vecdb/src/stamp/bytes.rs create mode 100644 crates/vecdb/src/stamp/conversions.rs create mode 100644 crates/vecdb/src/stamp/mod.rs create mode 100644 crates/vecdb/src/traits/any.rs create mode 100644 crates/vecdb/src/traits/any_exportable.rs create mode 100644 crates/vecdb/src/traits/any_readable.rs create mode 100644 crates/vecdb/src/traits/any_serializable.rs create mode 100644 crates/vecdb/src/traits/any_stored.rs create mode 100644 crates/vecdb/src/traits/any_with_schema.rs create mode 100644 crates/vecdb/src/traits/any_with_writer.rs create mode 100644 crates/vecdb/src/traits/formattable.rs create mode 100644 crates/vecdb/src/traits/importable.rs create mode 100644 crates/vecdb/src/traits/index.rs create mode 100644 crates/vecdb/src/traits/mod.rs create mode 100644 crates/vecdb/src/traits/printable.rs create mode 100644 crates/vecdb/src/traits/read_only_clone.rs create mode 100644 crates/vecdb/src/traits/readable.rs create mode 100644 crates/vecdb/src/traits/readable_option.rs create mode 100644 crates/vecdb/src/traits/storage_mode.rs create mode 100644 crates/vecdb/src/traits/stored.rs create mode 100644 crates/vecdb/src/traits/typed.rs create mode 100644 crates/vecdb/src/traits/value.rs create mode 100644 crates/vecdb/src/traits/value_strategy.rs create mode 100644 crates/vecdb/src/traits/writable.rs create mode 100644 crates/vecdb/src/variants/cached/any_vec.rs create mode 100644 crates/vecdb/src/variants/cached/budget.rs create mode 100644 crates/vecdb/src/variants/cached/clone.rs create mode 100644 crates/vecdb/src/variants/cached/cloneable.rs create mode 100644 crates/vecdb/src/variants/cached/mod.rs create mode 100644 crates/vecdb/src/variants/cached/read_only_clone.rs create mode 100644 crates/vecdb/src/variants/cached/readable.rs create mode 100644 crates/vecdb/src/variants/cached/typed.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/page/bytes.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/page/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/pages.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_only/any_vec.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_only/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_only/readable.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_only/typed.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/any_stored_vec.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/any_vec.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/readable.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/rollback.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/typed.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/read_write/writable.rs create mode 100644 crates/vecdb/src/variants/compressed/inner/strategy.rs create mode 100644 crates/vecdb/src/variants/compressed/lz4/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/lz4/strategy.rs create mode 100644 crates/vecdb/src/variants/compressed/lz4/value.rs create mode 100644 crates/vecdb/src/variants/compressed/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/strategy.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/trait.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/value/as_inner.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/value/as_inner_mut.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/value/from_inner.rs create mode 100644 crates/vecdb/src/variants/compressed/pco/value/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/sources/io.rs create mode 100644 crates/vecdb/src/variants/compressed/sources/mmap.rs create mode 100644 crates/vecdb/src/variants/compressed/sources/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/zstd/mod.rs create mode 100644 crates/vecdb/src/variants/compressed/zstd/strategy.rs create mode 100644 crates/vecdb/src/variants/compressed/zstd/value.rs create mode 100644 crates/vecdb/src/variants/eager/any_stored_vec.rs create mode 100644 crates/vecdb/src/variants/eager/any_vec.rs create mode 100644 crates/vecdb/src/variants/eager/compute/aggregates.rs create mode 100644 crates/vecdb/src/variants/eager/compute/arithmetic.rs create mode 100644 crates/vecdb/src/variants/eager/compute/cumulative.rs create mode 100644 crates/vecdb/src/variants/eager/compute/lookback.rs create mode 100644 crates/vecdb/src/variants/eager/compute/mod.rs create mode 100644 crates/vecdb/src/variants/eager/compute/statistics.rs create mode 100644 crates/vecdb/src/variants/eager/compute/transforms.rs create mode 100644 crates/vecdb/src/variants/eager/importable.rs create mode 100644 crates/vecdb/src/variants/eager/mod.rs create mode 100644 crates/vecdb/src/variants/eager/readable.rs create mode 100644 crates/vecdb/src/variants/eager/readable_cloneable.rs create mode 100644 crates/vecdb/src/variants/eager/stored.rs create mode 100644 crates/vecdb/src/variants/eager/typed.rs create mode 100644 crates/vecdb/src/variants/eager/writable.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/any_vec.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/clone.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/fold.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/readable.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/sparse.rs create mode 100644 crates/vecdb/src/variants/lazy/agg/typed.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/any_vec.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/avg.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/change.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/clone.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/op.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/rate.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/readable.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/sub.rs create mode 100644 crates/vecdb/src/variants/lazy/delta/typed.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/any_vec.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/read_only_clone.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/readable.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/transform/halve.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/transform/ident.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/transform/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/transform/negate.rs create mode 100644 crates/vecdb/src/variants/lazy/from1/typed.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/any_vec.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/readable.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/transform/divide.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/transform/minus.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/transform/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/transform/plus.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/transform/times.rs create mode 100644 crates/vecdb/src/variants/lazy/from2/typed.rs create mode 100644 crates/vecdb/src/variants/lazy/from3/any_vec.rs create mode 100644 crates/vecdb/src/variants/lazy/from3/mod.rs create mode 100644 crates/vecdb/src/variants/lazy/from3/readable.rs create mode 100644 crates/vecdb/src/variants/lazy/from3/typed.rs create mode 100644 crates/vecdb/src/variants/lazy/mod.rs create mode 100644 crates/vecdb/src/variants/macros.rs create mode 100644 crates/vecdb/src/variants/mod.rs create mode 100644 crates/vecdb/src/variants/raw/bytes/mod.rs create mode 100644 crates/vecdb/src/variants/raw/bytes/strategy/mod.rs create mode 100644 crates/vecdb/src/variants/raw/bytes/strategy/raw.rs create mode 100644 crates/vecdb/src/variants/raw/bytes/strategy/value.rs create mode 100644 crates/vecdb/src/variants/raw/bytes/value.rs create mode 100644 crates/vecdb/src/variants/raw/inner/mod.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_only/any_vec.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_only/mod.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_only/readable.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_only/typed.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/any_stored_vec.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/any_vec.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/change.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/mod.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/readable.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/rollback.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/typed.rs create mode 100644 crates/vecdb/src/variants/raw/inner/read_write/writable.rs create mode 100644 crates/vecdb/src/variants/raw/inner/strategy.rs create mode 100644 crates/vecdb/src/variants/raw/mod.rs create mode 100644 crates/vecdb/src/variants/raw/sources/io.rs create mode 100644 crates/vecdb/src/variants/raw/sources/mmap.rs create mode 100644 crates/vecdb/src/variants/raw/sources/mod.rs create mode 100644 crates/vecdb/src/variants/raw/sources/reader.rs create mode 100644 crates/vecdb/src/variants/raw/zerocopy/mod.rs create mode 100644 crates/vecdb/src/variants/raw/zerocopy/strategy/mod.rs create mode 100644 crates/vecdb/src/variants/raw/zerocopy/strategy/raw.rs create mode 100644 crates/vecdb/src/variants/raw/zerocopy/strategy/value.rs create mode 100644 crates/vecdb/src/variants/raw/zerocopy/value.rs create mode 100644 crates/vecdb/src/version/add.rs create mode 100644 crates/vecdb/src/version/bytes.rs create mode 100644 crates/vecdb/src/version/conversions.rs create mode 100644 crates/vecdb/src/version/display.rs create mode 100644 crates/vecdb/src/version/mod.rs create mode 100644 crates/vecdb/src/version/sum.rs create mode 100644 crates/vecdb/src/version/try_from_path.rs create mode 100644 crates/vecdb/tests/basic.rs create mode 100644 crates/vecdb/tests/basic2.rs create mode 100644 crates/vecdb/tests/comprehensive.rs create mode 100644 crates/vecdb/tests/compute_functions.rs create mode 100644 crates/vecdb/tests/concurrent_rw.rs create mode 100644 crates/vecdb/tests/consistency.rs create mode 100644 crates/vecdb/tests/cursor.rs create mode 100644 crates/vecdb/tests/cursor_chunk_size.rs create mode 100644 crates/vecdb/tests/integrity.rs create mode 100644 crates/vecdb/tests/iterators.rs create mode 100644 crates/vecdb/tests/non_copy.rs create mode 100644 crates/vecdb/tests/raw_last_page.rs create mode 100644 crates/vecdb/tests/raw_reader_cursor.rs create mode 100644 crates/vecdb/tests/rollback.rs create mode 100644 crates/vecdb/tests/rollback_truncation.rs create mode 100644 crates/vecdb/tests/scaling_bench.rs create mode 100644 crates/vecdb/tests/write_visibility.rs create mode 100644 crates/vecdb_derive/Cargo.toml create mode 100644 crates/vecdb_derive/README.md create mode 100644 crates/vecdb_derive/src/lib.rs create mode 100644 crates/vecdb_derive/tests/bytes.rs create mode 100644 crates/vecdb_derive/tests/bytes_generics.rs create mode 100644 crates/vecdb_derive/tests/pco.rs create mode 100644 crates/vecdb_derive/tests/pco_generics.rs diff --git a/Cargo.lock b/Cargo.lock index e98d777cf..d4d318c38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,19 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -62,6 +75,15 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -71,6 +93,62 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -85,9 +163,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -315,6 +393,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brk" version = "0.3.6" @@ -658,6 +745,7 @@ dependencies = [ "byteview", "fjall", "rustc-hash", + "tempfile", ] [[package]] @@ -773,15 +861,21 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "byteview" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" +checksum = "0d74937895e761d5984206f82ca8ff7725d0fd8021921011921894b41ab1b9f7" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -804,6 +898,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -818,16 +923,68 @@ dependencies = [ ] [[package]] -name = "clang-sys" -version = "1.8.1" +name = "ciborium" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" dependencies = [ "glob", "libc", "libloading", ] +[[package]] +name = "clap" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "color-eyre" version = "0.6.5" @@ -861,6 +1018,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "compare" version = "0.0.6" @@ -887,6 +1050,12 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "convert_case" version = "0.10.0" @@ -988,6 +1157,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -997,6 +1175,41 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -1069,6 +1282,15 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cty" version = "0.2.2" @@ -1149,6 +1371,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + [[package]] name = "dirs" version = "6.0.0" @@ -1172,13 +1405,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -1225,9 +1458,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "enum_dispatch" @@ -1241,6 +1474,28 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1273,6 +1528,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -1290,7 +1557,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fjall" -version = "3.1.8" +version = "0.3.6" dependencies = [ "byteorder-lite", "byteview", @@ -1299,7 +1566,10 @@ dependencies = [ "log", "lsm-tree", "lz4_flex", + "nanoid", + "rand 0.10.2", "tempfile", + "test-log", "xxhash-rust", ] @@ -1400,6 +1670,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.33" @@ -1450,6 +1726,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -1458,7 +1746,8 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -1550,9 +1839,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1593,6 +1882,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1843,6 +2141,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.13.0" @@ -1860,9 +2164,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ "defmt", "jiff-core", @@ -1885,9 +2189,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ "jiff-core", "proc-macro2", @@ -2026,20 +2330,27 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lsm-tree" -version = "3.1.8" +version = "0.3.6" dependencies = [ "byteorder-lite", + "bytes", "byteview", + "criterion", "crossbeam-skiplist", "enum_dispatch", + "fs_extra", "interval-heap", "log", "lz4_flex", + "nanoid", "quick_cache", + "rand 0.9.5", "rustc-hash", "self_cell", "sfa", + "strum", "tempfile", + "test-log", "varint-rs", "xxhash-rust", ] @@ -2049,8 +2360,14 @@ name = "lz4_flex" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "twox-hash", + "regex-automata", ] [[package]] @@ -2129,6 +2446,35 @@ dependencies = [ "serde_json", ] +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nanoid" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" +dependencies = [ + "rand 0.8.7", +] + [[package]] name = "nom" version = "7.1.3" @@ -2139,6 +2485,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2187,6 +2542,18 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" @@ -2203,6 +2570,16 @@ dependencies = [ "supports-color 3.0.2", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2364,6 +2741,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -2373,6 +2759,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "quick_cache" version = "0.6.24" @@ -2385,9 +2791,7 @@ dependencies = [ [[package]] name = "quickmatch" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6abd98fde5c9aa23590316caedc9e90be29da035d33905cdae71a54f034aaac" +version = "0.3.6" dependencies = [ "rustc-hash", ] @@ -2401,17 +2805,102 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rancor" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daff8b7b3ccf5f7ba270b3e7a0a4d4c701c5797e38dec27c7e2c3dbb830fed1c" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_xoshiro" @@ -2419,7 +2908,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2433,7 +2922,7 @@ dependencies = [ [[package]] name = "rawdb" -version = "0.10.4" +version = "0.3.6" dependencies = [ "libc", "log", @@ -2441,6 +2930,7 @@ dependencies = [ "parking_lot", "rayon", "smallvec", + "tempfile", "thiserror", ] @@ -2533,6 +3023,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rend" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "663ba70707f96e871406fe10d68128412e619b06d1d47cb91c3a4c6501176240" + [[package]] name = "ring" version = "0.17.14" @@ -2547,6 +3043,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" +dependencies = [ + "bytes", + "hashbrown 0.17.1", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "rlimit" version = "0.11.0" @@ -2592,9 +3117,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "log", "once_cell", @@ -2648,9 +3173,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "dyn-clone", "indexmap", @@ -2662,14 +3187,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2752,13 +3277,13 @@ dependencies = [ [[package]] name = "serde_derive_internals" -version = "0.29.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2831,6 +3356,17 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2858,6 +3394,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -2880,6 +3422,45 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonic-number" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + [[package]] name = "spin" version = "0.9.9" @@ -2993,6 +3574,38 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "test-log" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b9c218384242b5c89b68303ab6f6fc53a312d923f0c14dc6bb860c6aeee40f1" +dependencies = [ + "env_logger", + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-core" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c26ef8b00e4d382e59f6a8ddb3cd790b3a5bb29f21a358a9a69ea2f29f13f27b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "test-log-macros" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "944ad38adcbb71eaa682c56bceeb079e4ca82b4b3edc2a0fde5cb297b77dac8d" +dependencies = [ + "syn 2.0.119", + "test-log-core", +] + [[package]] name = "thiserror" version = "2.0.19" @@ -3024,9 +3637,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -3062,6 +3675,31 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -3078,13 +3716,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -3102,9 +3740,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", @@ -3126,9 +3764,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -3248,9 +3886,15 @@ version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", "thread_local", + "tracing", "tracing-core", + "tracing-log", ] [[package]] @@ -3260,10 +3904,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" [[package]] -name = "twox-hash" -version = "2.1.3" +name = "typenum" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" @@ -3346,6 +3990,22 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3360,7 +4020,7 @@ checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96" [[package]] name = "vecdb" -version = "0.10.4" +version = "0.3.6" dependencies = [ "itoa", "libc", @@ -3373,6 +4033,9 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", + "sonic-rs", + "tempfile", "thiserror", "vecdb_derive", "zerocopy", @@ -3381,10 +4044,12 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.10.4" +version = "0.3.6" dependencies = [ "quote", "syn 2.0.119", + "tempfile", + "vecdb", ] [[package]] @@ -3409,6 +4074,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -3666,6 +4340,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index 15644ab32..0ca1fcbf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,33 +59,40 @@ brk_traversable = { version = "0.3.6", path = "crates/brk_traversable", features brk_traversable_derive = { version = "0.3.6", path = "crates/brk_traversable_derive" } brk_types = { version = "0.3.6", path = "crates/brk_types" } brk_website = { version = "0.3.6", path = "crates/brk_website" } -byteview = "0.10.1" +byteview = "0.10.2" color-eyre = "0.6.5" corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false } corepc-types = { version = "0.15.0", features = ["std"], default-features = false } derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] } -fjall = { path = "../fjall", version = "3.1.8" } +fjall = { path = "crates/fjall", version = "0.3.6" } indexmap = { version = "2.14.0", features = ["serde"] } -jiff = { version = "0.2.34", features = ["perf-inline", "tz-system"], default-features = false } +jiff = { version = "0.2.35", features = ["perf-inline", "tz-system"], default-features = false } +libc = "0.2" +log = "0.4.33" +lsm-tree = { path = "crates/lsm-tree", version = "0.3.6", default-features = false } +lz4_flex = { version = "0.13.1", default-features = false } owo-colors = "4.3.0" parking_lot = "0.12.5" pco = "1.0.2" +quickmatch = { path = "crates/quickmatch", version = "0.3.6" } +rawdb = { path = "crates/rawdb", version = "0.3.6" } rayon = "1.12.0" rapidhash = "4.5.1" rustc-hash = "2.1.3" -schemars = { version = "1.2.1", features = ["indexmap2"] } +schemars = { version = "1.2.2", features = ["indexmap2"] } serde = "1.0.229" serde_bytes = "0.11.19" serde_derive = "1.0.229" serde_json = { version = "1.0.151", features = ["float_roundtrip", "preserve_order"] } smallvec = "1.15.2" +tempfile = "3.27.0" tokio = { version = "1.53.1", features = ["rt-multi-thread"] } tower-http = { version = "0.7.0", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] } tower-layer = "0.3" tracing = { version = "0.1", default-features = false, features = ["std"] } ureq = { version = "3.3.0", features = ["json"] } -# vecdb = { version = "0.10.4", features = ["derive", "serde_json", "pco", "schemars"] } -vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } +vecdb = { path = "crates/vecdb", version = "0.3.6", features = ["derive", "serde_json", "pco", "schemars"] } +vecdb_derive = { path = "crates/vecdb_derive", version = "0.3.6" } [workspace.metadata.release] diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 58579d0a6..f6c0e5e0c 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -25,7 +25,7 @@ owo-colors = { workspace = true } tracing = { workspace = true } serde = { workspace = true } tokio = { workspace = true } -toml = "1.1.3" +toml = "1.1.4" vecdb = { workspace = true } [[bin]] diff --git a/crates/brk_client/src/lib.rs b/crates/brk_client/src/lib.rs index b62fe41ef..0baf969d7 100644 --- a/crates/brk_client/src/lib.rs +++ b/crates/brk_client/src/lib.rs @@ -8,13 +8,12 @@ #![allow(clippy::useless_format)] #![allow(clippy::unnecessary_to_owned)] -use std::str::FromStr; -use std::sync::Arc; -use std::ops::{Bound, RangeBounds}; -use serde::de::DeserializeOwned; pub use brk_cohort::*; pub use brk_types::*; - +use serde::de::DeserializeOwned; +use std::ops::{Bound, RangeBounds}; +use std::str::FromStr; +use std::sync::Arc; /// Error type for BRK client operations. #[derive(Debug)] @@ -50,13 +49,19 @@ pub struct AddressHashPrefix { /// Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{addr_type}/{prefix}`. pub fn address_payload_hash_prefix(payload: &[u8], nibbles: usize) -> Result { if payload.is_empty() { - return Err(BrkError { message: "Expected a non-empty address payload".to_string() }); + return Err(BrkError { + message: "Expected a non-empty address payload".to_string(), + }); } if payload.len() > 65 { - return Err(BrkError { message: "Expected at most 65 address payload bytes".to_string() }); + return Err(BrkError { + message: "Expected at most 65 address payload bytes".to_string(), + }); } if !(1..=16).contains(&nibbles) { - return Err(BrkError { message: "Expected hash-prefix length from 1 to 16 hex nibbles".to_string() }); + return Err(BrkError { + message: "Expected hash-prefix length from 1 to 16 hex nibbles".to_string(), + }); } Ok(format!("{:016x}", rapidhash::v3::rapidhash_v3(payload))[..nibbles].to_string()) } @@ -69,8 +74,12 @@ fn validate_address_payload_for_type(addr_type: OutputType, payload: &[u8]) -> R OutputType::P2PKH | OutputType::P2SH | OutputType::P2WPKH => &[20], OutputType::P2WSH | OutputType::P2TR => &[32], OutputType::P2MS | OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => { - return Err(BrkError { message: format!("Unsupported address type for address payload hash-prefix: {addr_type:?}") }); - }, + return Err(BrkError { + message: format!( + "Unsupported address type for address payload hash-prefix: {addr_type:?}" + ), + }); + } }; let addr_type = address_payload_type_path(addr_type)?; @@ -80,7 +89,9 @@ fn validate_address_payload_for_type(addr_type: OutputType, payload: &[u8]) -> R .map(ToString::to_string) .collect::>() .join(" or "); - return Err(BrkError { message: format!("Expected {addr_type} address payload length {joined} bytes") }); + return Err(BrkError { + message: format!("Expected {addr_type} address payload length {joined} bytes"), + }); } Ok(()) @@ -96,17 +107,25 @@ fn address_payload_type_path(addr_type: OutputType) -> Result<&'static str> { OutputType::P2WSH => Ok("v0_p2wsh"), OutputType::P2TR => Ok("v1_p2tr"), OutputType::P2MS | OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => { - Err(BrkError { message: format!("Unsupported address type for address payload hash-prefix: {addr_type:?}") }) - }, + Err(BrkError { + message: format!( + "Unsupported address type for address payload hash-prefix: {addr_type:?}" + ), + }) + } } } /// Decode a mainnet Bitcoin address into the BRK address type and raw payload bytes. pub fn decode_address_payload(address: &str) -> Result { if address.is_empty() { - return Err(BrkError { message: "Expected an address string".to_string() }); + return Err(BrkError { + message: "Expected an address string".to_string(), + }); } - let addr_bytes = AddrBytes::from_str(address).map_err(|e| BrkError { message: e.to_string() })?; + let addr_bytes = AddrBytes::from_str(address).map_err(|e| BrkError { + message: e.to_string(), + })?; let addr_type = OutputType::from(&addr_bytes); Ok(AddressPayload { @@ -150,7 +169,10 @@ pub struct BrkClientBase { impl BrkClientBase { /// Create a new client with the given base URL. pub fn new(base_url: impl Into) -> Self { - Self::with_options(BrkClientOptions { base_url: base_url.into(), ..Default::default() }) + Self::with_options(BrkClientOptions { + base_url: base_url.into(), + ..Default::default() + }) } /// Create a new client with options. @@ -171,68 +193,93 @@ impl BrkClientBase { /// Make a GET request and deserialize JSON response. pub fn get_json(&self, path: &str) -> Result { - self.agent.get(&self.url(path)) + self.agent + .get(&self.url(path)) .call() .and_then(|mut r| r.body_mut().read_json()) - .map_err(|e| BrkError { message: e.to_string() }) + .map_err(|e| BrkError { + message: e.to_string(), + }) } /// Make a GET request and return raw text response. pub fn get_text(&self, path: &str) -> Result { - self.agent.get(&self.url(path)) + self.agent + .get(&self.url(path)) .call() .and_then(|mut r| r.body_mut().read_to_string()) - .map_err(|e| BrkError { message: e.to_string() }) + .map_err(|e| BrkError { + message: e.to_string(), + }) } /// Make a GET request and return raw bytes response. pub fn get_bytes(&self, path: &str) -> Result> { - self.agent.get(&self.url(path)) + self.agent + .get(&self.url(path)) .call() .and_then(|mut r| r.body_mut().read_to_vec()) - .map_err(|e| BrkError { message: e.to_string() }) + .map_err(|e| BrkError { + message: e.to_string(), + }) } /// Make a POST request and deserialize JSON response. pub fn post_json(&self, path: &str, body: &str) -> Result { - self.agent.post(&self.url(path)) + self.agent + .post(&self.url(path)) .send(body) .and_then(|mut r| r.body_mut().read_json()) - .map_err(|e| BrkError { message: e.to_string() }) + .map_err(|e| BrkError { + message: e.to_string(), + }) } /// Make a POST request and return raw text response. pub fn post_text(&self, path: &str, body: &str) -> Result { - self.agent.post(&self.url(path)) + self.agent + .post(&self.url(path)) .send(body) .and_then(|mut r| r.body_mut().read_to_string()) - .map_err(|e| BrkError { message: e.to_string() }) + .map_err(|e| BrkError { + message: e.to_string(), + }) } /// Make a POST request and return raw bytes response. pub fn post_bytes(&self, path: &str, body: &str) -> Result> { - self.agent.post(&self.url(path)) + self.agent + .post(&self.url(path)) .send(body) .and_then(|mut r| r.body_mut().read_to_vec()) - .map_err(|e| BrkError { message: e.to_string() }) + .map_err(|e| BrkError { + message: e.to_string(), + }) } } /// Build series name with suffix. #[inline] fn _m(acc: &str, s: &str) -> String { - if s.is_empty() { acc.to_string() } - else if acc.is_empty() { s.to_string() } - else { format!("{acc}_{s}") } + if s.is_empty() { + acc.to_string() + } else if acc.is_empty() { + s.to_string() + } else { + format!("{acc}_{s}") + } } /// Build series name with prefix. #[inline] fn _p(prefix: &str, acc: &str) -> String { - if acc.is_empty() { prefix.to_string() } else { format!("{prefix}_{acc}") } + if acc.is_empty() { + prefix.to_string() + } else { + format!("{prefix}_{acc}") + } } - /// Non-generic trait for series patterns (usable in collections). pub trait AnySeriesPattern { /// Get the series name. @@ -248,7 +295,6 @@ pub trait SeriesPattern: AnySeriesPattern { fn get(&self, index: Index) -> Option>; } - /// Shared endpoint configuration. #[derive(Clone)] struct EndpointConfig { @@ -261,7 +307,13 @@ struct EndpointConfig { impl EndpointConfig { fn new(client: Arc, name: Arc, index: Index) -> Self { - Self { client, name, index, start: None, end: None } + Self { + client, + name, + index, + start: None, + end: None, + } } fn path(&self) -> String { @@ -270,11 +322,21 @@ impl EndpointConfig { fn build_path(&self, format: Option<&str>) -> String { let mut params = Vec::new(); - if let Some(s) = self.start { params.push(format!("start={}", s)); } - if let Some(e) = self.end { params.push(format!("end={}", e)); } - if let Some(fmt) = format { params.push(format!("format={}", fmt)); } + if let Some(s) = self.start { + params.push(format!("start={}", s)); + } + if let Some(e) = self.end { + params.push(format!("end={}", e)); + } + if let Some(fmt) = format { + params.push(format!("format={}", fmt)); + } let p = self.path(); - if params.is_empty() { p } else { format!("{}?{}", p, params.join("&")) } + if params.is_empty() { + p + } else { + format!("{}?{}", p, params.join("&")) + } } fn get_json(&self, format: Option<&str>) -> Result { @@ -286,11 +348,19 @@ impl EndpointConfig { } fn get_len(&self) -> Result { - self.client.get_json(&format!("/api/series/{}/{}/len", self.name, self.index.name())) + self.client.get_json(&format!( + "/api/series/{}/{}/len", + self.name, + self.index.name() + )) } fn get_version(&self) -> Result { - self.client.get_json(&format!("/api/series/{}/{}/version", self.name, self.index.name())) + self.client.get_json(&format!( + "/api/series/{}/{}/version", + self.name, + self.index.name() + )) } } @@ -322,14 +392,20 @@ pub type DateSeriesEndpoint = SeriesEndpoint>; impl SeriesEndpoint { pub fn new(client: Arc, name: Arc, index: Index) -> Self { - Self { config: EndpointConfig::new(client, name, index), _marker: std::marker::PhantomData } + Self { + config: EndpointConfig::new(client, name, index), + _marker: std::marker::PhantomData, + } } /// Select a specific index position. pub fn get(mut self, index: usize) -> SingleItemBuilder { self.config.start = Some(index as i64); self.config.end = Some(index as i64 + 1); - SingleItemBuilder { config: self.config, _marker: std::marker::PhantomData } + SingleItemBuilder { + config: self.config, + _marker: std::marker::PhantomData, + } } /// Select a range using Rust range syntax. @@ -351,7 +427,10 @@ impl SeriesEndpoint { Bound::Excluded(&n) => Some(n as i64), Bound::Unbounded => None, }; - RangeBuilder { config: self.config, _marker: std::marker::PhantomData } + RangeBuilder { + config: self.config, + _marker: std::marker::PhantomData, + } } /// Take the first n items. @@ -366,13 +445,19 @@ impl SeriesEndpoint { } else { self.config.start = Some(-(n as i64)); } - RangeBuilder { config: self.config, _marker: std::marker::PhantomData } + RangeBuilder { + config: self.config, + _marker: std::marker::PhantomData, + } } /// Skip the first n items. Chain with `take(n)` to get a range. pub fn skip(mut self, n: usize) -> SkippedBuilder { self.config.start = Some(n as i64); - SkippedBuilder { config: self.config, _marker: std::marker::PhantomData } + SkippedBuilder { + config: self.config, + _marker: std::marker::PhantomData, + } } /// Fetch all data as parsed JSON. @@ -424,7 +509,11 @@ impl SeriesEndpoint> { } /// Select a timestamp range (works for all date-based indexes including sub-daily). - pub fn timestamp_range(self, start: Timestamp, end: Timestamp) -> RangeBuilder> { + pub fn timestamp_range( + self, + start: Timestamp, + end: Timestamp, + ) -> RangeBuilder> { let s = self.config.index.timestamp_to_index(start).unwrap_or(0); let e = self.config.index.timestamp_to_index(end).unwrap_or(0); self.range(s..e) @@ -466,7 +555,10 @@ impl SkippedBuilder { pub fn take(mut self, n: usize) -> RangeBuilder { let start = self.config.start.unwrap_or(0); self.config.end = Some(start + n as i64); - RangeBuilder { config: self.config, _marker: std::marker::PhantomData } + RangeBuilder { + config: self.config, + _marker: std::marker::PhantomData, + } } /// Fetch from the skipped position to the end. @@ -501,10 +593,42 @@ impl RangeBuilder { } } - // Static index arrays -const _I1: &[Index] = &[Index::Minute10, Index::Minute30, Index::Hour1, Index::Hour4, Index::Hour12, Index::Day1, Index::Day3, Index::Week1, Index::Month1, Index::Month3, Index::Month6, Index::Year1, Index::Year10, Index::Halving, Index::Epoch, Index::Height]; -const _I2: &[Index] = &[Index::Minute10, Index::Minute30, Index::Hour1, Index::Hour4, Index::Hour12, Index::Day1, Index::Day3, Index::Week1, Index::Month1, Index::Month3, Index::Month6, Index::Year1, Index::Year10, Index::Halving, Index::Epoch]; +const _I1: &[Index] = &[ + Index::Minute10, + Index::Minute30, + Index::Hour1, + Index::Hour4, + Index::Hour12, + Index::Day1, + Index::Day3, + Index::Week1, + Index::Month1, + Index::Month3, + Index::Month6, + Index::Year1, + Index::Year10, + Index::Halving, + Index::Epoch, + Index::Height, +]; +const _I2: &[Index] = &[ + Index::Minute10, + Index::Minute30, + Index::Hour1, + Index::Hour4, + Index::Hour12, + Index::Day1, + Index::Day3, + Index::Week1, + Index::Month1, + Index::Month3, + Index::Month6, + Index::Year1, + Index::Year10, + Index::Halving, + Index::Epoch, +]; const _I3: &[Index] = &[Index::Minute10]; const _I4: &[Index] = &[Index::Minute30]; const _I5: &[Index] = &[Index::Hour1]; @@ -545,535 +669,1753 @@ fn _ep(c: &Arc, n: &Arc, i: Index) -> S } #[inline] -fn _dep(c: &Arc, n: &Arc, i: Index) -> DateSeriesEndpoint { +fn _dep( + c: &Arc, + n: &Arc, + i: Index, +) -> DateSeriesEndpoint { DateSeriesEndpoint::new(c.clone(), n.clone(), i) } // Index accessor structs -pub struct SeriesPattern1By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern1By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern1By { - pub fn minute10(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Minute10) } - pub fn minute30(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Minute30) } - pub fn hour1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour1) } - pub fn hour4(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour4) } - pub fn hour12(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour12) } - pub fn day1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Day1) } - pub fn day3(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Day3) } - pub fn week1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Week1) } - pub fn month1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month1) } - pub fn month3(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month3) } - pub fn month6(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month6) } - pub fn year1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Year1) } - pub fn year10(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Year10) } - pub fn halving(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Halving) } - pub fn epoch(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Epoch) } - pub fn height(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Height) } + pub fn minute10(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Minute10) + } + pub fn minute30(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Minute30) + } + pub fn hour1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour1) + } + pub fn hour4(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour4) + } + pub fn hour12(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour12) + } + pub fn day1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Day1) + } + pub fn day3(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Day3) + } + pub fn week1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Week1) + } + pub fn month1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month1) + } + pub fn month3(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month3) + } + pub fn month6(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month6) + } + pub fn year1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Year1) + } + pub fn year10(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Year10) + } + pub fn halving(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Halving) + } + pub fn epoch(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Epoch) + } + pub fn height(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Height) + } } -pub struct SeriesPattern1 { name: Arc, pub by: SeriesPattern1By } +pub struct SeriesPattern1 { + name: Arc, + pub by: SeriesPattern1By, +} impl SeriesPattern1 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern1By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern1By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern1 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I1 } } -impl SeriesPattern for SeriesPattern1 { fn get(&self, index: Index) -> Option> { _I1.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern1 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I1 + } +} +impl SeriesPattern for SeriesPattern1 { + fn get(&self, index: Index) -> Option> { + _I1.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern2By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern2By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern2By { - pub fn minute10(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Minute10) } - pub fn minute30(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Minute30) } - pub fn hour1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour1) } - pub fn hour4(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour4) } - pub fn hour12(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour12) } - pub fn day1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Day1) } - pub fn day3(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Day3) } - pub fn week1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Week1) } - pub fn month1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month1) } - pub fn month3(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month3) } - pub fn month6(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month6) } - pub fn year1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Year1) } - pub fn year10(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Year10) } - pub fn halving(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Halving) } - pub fn epoch(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Epoch) } + pub fn minute10(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Minute10) + } + pub fn minute30(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Minute30) + } + pub fn hour1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour1) + } + pub fn hour4(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour4) + } + pub fn hour12(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour12) + } + pub fn day1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Day1) + } + pub fn day3(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Day3) + } + pub fn week1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Week1) + } + pub fn month1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month1) + } + pub fn month3(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month3) + } + pub fn month6(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month6) + } + pub fn year1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Year1) + } + pub fn year10(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Year10) + } + pub fn halving(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Halving) + } + pub fn epoch(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Epoch) + } } -pub struct SeriesPattern2 { name: Arc, pub by: SeriesPattern2By } +pub struct SeriesPattern2 { + name: Arc, + pub by: SeriesPattern2By, +} impl SeriesPattern2 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern2By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern2By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern2 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I2 } } -impl SeriesPattern for SeriesPattern2 { fn get(&self, index: Index) -> Option> { _I2.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern2 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I2 + } +} +impl SeriesPattern for SeriesPattern2 { + fn get(&self, index: Index) -> Option> { + _I2.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern3By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern3By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern3By { - pub fn minute10(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Minute10) } + pub fn minute10(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Minute10) + } } -pub struct SeriesPattern3 { name: Arc, pub by: SeriesPattern3By } +pub struct SeriesPattern3 { + name: Arc, + pub by: SeriesPattern3By, +} impl SeriesPattern3 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern3By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern3By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern3 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I3 } } -impl SeriesPattern for SeriesPattern3 { fn get(&self, index: Index) -> Option> { _I3.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern3 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I3 + } +} +impl SeriesPattern for SeriesPattern3 { + fn get(&self, index: Index) -> Option> { + _I3.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern4By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern4By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern4By { - pub fn minute30(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Minute30) } + pub fn minute30(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Minute30) + } } -pub struct SeriesPattern4 { name: Arc, pub by: SeriesPattern4By } +pub struct SeriesPattern4 { + name: Arc, + pub by: SeriesPattern4By, +} impl SeriesPattern4 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern4By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern4By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern4 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I4 } } -impl SeriesPattern for SeriesPattern4 { fn get(&self, index: Index) -> Option> { _I4.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern4 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I4 + } +} +impl SeriesPattern for SeriesPattern4 { + fn get(&self, index: Index) -> Option> { + _I4.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern5By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern5By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern5By { - pub fn hour1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour1) } + pub fn hour1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour1) + } } -pub struct SeriesPattern5 { name: Arc, pub by: SeriesPattern5By } +pub struct SeriesPattern5 { + name: Arc, + pub by: SeriesPattern5By, +} impl SeriesPattern5 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern5By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern5By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern5 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I5 } } -impl SeriesPattern for SeriesPattern5 { fn get(&self, index: Index) -> Option> { _I5.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern5 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I5 + } +} +impl SeriesPattern for SeriesPattern5 { + fn get(&self, index: Index) -> Option> { + _I5.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern6By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern6By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern6By { - pub fn hour4(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour4) } + pub fn hour4(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour4) + } } -pub struct SeriesPattern6 { name: Arc, pub by: SeriesPattern6By } +pub struct SeriesPattern6 { + name: Arc, + pub by: SeriesPattern6By, +} impl SeriesPattern6 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern6By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern6By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern6 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I6 } } -impl SeriesPattern for SeriesPattern6 { fn get(&self, index: Index) -> Option> { _I6.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern6 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I6 + } +} +impl SeriesPattern for SeriesPattern6 { + fn get(&self, index: Index) -> Option> { + _I6.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern7By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern7By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern7By { - pub fn hour12(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Hour12) } + pub fn hour12(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Hour12) + } } -pub struct SeriesPattern7 { name: Arc, pub by: SeriesPattern7By } +pub struct SeriesPattern7 { + name: Arc, + pub by: SeriesPattern7By, +} impl SeriesPattern7 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern7By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern7By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern7 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I7 } } -impl SeriesPattern for SeriesPattern7 { fn get(&self, index: Index) -> Option> { _I7.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern7 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I7 + } +} +impl SeriesPattern for SeriesPattern7 { + fn get(&self, index: Index) -> Option> { + _I7.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern8By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern8By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern8By { - pub fn day1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Day1) } + pub fn day1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Day1) + } } -pub struct SeriesPattern8 { name: Arc, pub by: SeriesPattern8By } +pub struct SeriesPattern8 { + name: Arc, + pub by: SeriesPattern8By, +} impl SeriesPattern8 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern8By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern8By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern8 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I8 } } -impl SeriesPattern for SeriesPattern8 { fn get(&self, index: Index) -> Option> { _I8.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern8 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I8 + } +} +impl SeriesPattern for SeriesPattern8 { + fn get(&self, index: Index) -> Option> { + _I8.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern9By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern9By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern9By { - pub fn day3(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Day3) } + pub fn day3(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Day3) + } } -pub struct SeriesPattern9 { name: Arc, pub by: SeriesPattern9By } +pub struct SeriesPattern9 { + name: Arc, + pub by: SeriesPattern9By, +} impl SeriesPattern9 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern9By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern9By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern9 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I9 } } -impl SeriesPattern for SeriesPattern9 { fn get(&self, index: Index) -> Option> { _I9.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern9 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I9 + } +} +impl SeriesPattern for SeriesPattern9 { + fn get(&self, index: Index) -> Option> { + _I9.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern10By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern10By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern10By { - pub fn week1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Week1) } + pub fn week1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Week1) + } } -pub struct SeriesPattern10 { name: Arc, pub by: SeriesPattern10By } +pub struct SeriesPattern10 { + name: Arc, + pub by: SeriesPattern10By, +} impl SeriesPattern10 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern10By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern10By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern10 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I10 } } -impl SeriesPattern for SeriesPattern10 { fn get(&self, index: Index) -> Option> { _I10.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern10 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I10 + } +} +impl SeriesPattern for SeriesPattern10 { + fn get(&self, index: Index) -> Option> { + _I10.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern11By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern11By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern11By { - pub fn month1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month1) } + pub fn month1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month1) + } } -pub struct SeriesPattern11 { name: Arc, pub by: SeriesPattern11By } +pub struct SeriesPattern11 { + name: Arc, + pub by: SeriesPattern11By, +} impl SeriesPattern11 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern11By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern11By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern11 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I11 } } -impl SeriesPattern for SeriesPattern11 { fn get(&self, index: Index) -> Option> { _I11.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern11 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I11 + } +} +impl SeriesPattern for SeriesPattern11 { + fn get(&self, index: Index) -> Option> { + _I11.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern12By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern12By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern12By { - pub fn month3(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month3) } + pub fn month3(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month3) + } } -pub struct SeriesPattern12 { name: Arc, pub by: SeriesPattern12By } +pub struct SeriesPattern12 { + name: Arc, + pub by: SeriesPattern12By, +} impl SeriesPattern12 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern12By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern12By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern12 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I12 } } -impl SeriesPattern for SeriesPattern12 { fn get(&self, index: Index) -> Option> { _I12.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern12 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I12 + } +} +impl SeriesPattern for SeriesPattern12 { + fn get(&self, index: Index) -> Option> { + _I12.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern13By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern13By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern13By { - pub fn month6(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Month6) } + pub fn month6(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Month6) + } } -pub struct SeriesPattern13 { name: Arc, pub by: SeriesPattern13By } +pub struct SeriesPattern13 { + name: Arc, + pub by: SeriesPattern13By, +} impl SeriesPattern13 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern13By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern13By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern13 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I13 } } -impl SeriesPattern for SeriesPattern13 { fn get(&self, index: Index) -> Option> { _I13.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern13 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I13 + } +} +impl SeriesPattern for SeriesPattern13 { + fn get(&self, index: Index) -> Option> { + _I13.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern14By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern14By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern14By { - pub fn year1(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Year1) } + pub fn year1(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Year1) + } } -pub struct SeriesPattern14 { name: Arc, pub by: SeriesPattern14By } +pub struct SeriesPattern14 { + name: Arc, + pub by: SeriesPattern14By, +} impl SeriesPattern14 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern14By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern14By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern14 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I14 } } -impl SeriesPattern for SeriesPattern14 { fn get(&self, index: Index) -> Option> { _I14.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern14 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I14 + } +} +impl SeriesPattern for SeriesPattern14 { + fn get(&self, index: Index) -> Option> { + _I14.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern15By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern15By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern15By { - pub fn year10(&self) -> DateSeriesEndpoint { _dep(&self.client, &self.name, Index::Year10) } + pub fn year10(&self) -> DateSeriesEndpoint { + _dep(&self.client, &self.name, Index::Year10) + } } -pub struct SeriesPattern15 { name: Arc, pub by: SeriesPattern15By } +pub struct SeriesPattern15 { + name: Arc, + pub by: SeriesPattern15By, +} impl SeriesPattern15 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern15By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern15By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern15 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I15 } } -impl SeriesPattern for SeriesPattern15 { fn get(&self, index: Index) -> Option> { _I15.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern15 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I15 + } +} +impl SeriesPattern for SeriesPattern15 { + fn get(&self, index: Index) -> Option> { + _I15.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern16By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern16By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern16By { - pub fn halving(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Halving) } + pub fn halving(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Halving) + } } -pub struct SeriesPattern16 { name: Arc, pub by: SeriesPattern16By } +pub struct SeriesPattern16 { + name: Arc, + pub by: SeriesPattern16By, +} impl SeriesPattern16 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern16By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern16By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern16 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I16 } } -impl SeriesPattern for SeriesPattern16 { fn get(&self, index: Index) -> Option> { _I16.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern16 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I16 + } +} +impl SeriesPattern for SeriesPattern16 { + fn get(&self, index: Index) -> Option> { + _I16.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern17By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern17By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern17By { - pub fn epoch(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Epoch) } + pub fn epoch(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Epoch) + } } -pub struct SeriesPattern17 { name: Arc, pub by: SeriesPattern17By } +pub struct SeriesPattern17 { + name: Arc, + pub by: SeriesPattern17By, +} impl SeriesPattern17 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern17By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern17By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern17 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I17 } } -impl SeriesPattern for SeriesPattern17 { fn get(&self, index: Index) -> Option> { _I17.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern17 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I17 + } +} +impl SeriesPattern for SeriesPattern17 { + fn get(&self, index: Index) -> Option> { + _I17.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern18By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern18By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern18By { - pub fn height(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::Height) } + pub fn height(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::Height) + } } -pub struct SeriesPattern18 { name: Arc, pub by: SeriesPattern18By } +pub struct SeriesPattern18 { + name: Arc, + pub by: SeriesPattern18By, +} impl SeriesPattern18 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern18By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern18By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern18 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I18 } } -impl SeriesPattern for SeriesPattern18 { fn get(&self, index: Index) -> Option> { _I18.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern18 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I18 + } +} +impl SeriesPattern for SeriesPattern18 { + fn get(&self, index: Index) -> Option> { + _I18.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern19By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern19By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern19By { - pub fn tx_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::TxIndex) } + pub fn tx_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::TxIndex) + } } -pub struct SeriesPattern19 { name: Arc, pub by: SeriesPattern19By } +pub struct SeriesPattern19 { + name: Arc, + pub by: SeriesPattern19By, +} impl SeriesPattern19 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern19By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern19By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern19 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I19 } } -impl SeriesPattern for SeriesPattern19 { fn get(&self, index: Index) -> Option> { _I19.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern19 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I19 + } +} +impl SeriesPattern for SeriesPattern19 { + fn get(&self, index: Index) -> Option> { + _I19.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern20By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern20By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern20By { - pub fn txin_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::TxInIndex) } + pub fn txin_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::TxInIndex) + } } -pub struct SeriesPattern20 { name: Arc, pub by: SeriesPattern20By } +pub struct SeriesPattern20 { + name: Arc, + pub by: SeriesPattern20By, +} impl SeriesPattern20 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern20By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern20By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern20 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I20 } } -impl SeriesPattern for SeriesPattern20 { fn get(&self, index: Index) -> Option> { _I20.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern20 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I20 + } +} +impl SeriesPattern for SeriesPattern20 { + fn get(&self, index: Index) -> Option> { + _I20.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern21By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern21By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern21By { - pub fn txout_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::TxOutIndex) } + pub fn txout_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::TxOutIndex) + } } -pub struct SeriesPattern21 { name: Arc, pub by: SeriesPattern21By } +pub struct SeriesPattern21 { + name: Arc, + pub by: SeriesPattern21By, +} impl SeriesPattern21 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern21By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern21By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern21 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I21 } } -impl SeriesPattern for SeriesPattern21 { fn get(&self, index: Index) -> Option> { _I21.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern21 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I21 + } +} +impl SeriesPattern for SeriesPattern21 { + fn get(&self, index: Index) -> Option> { + _I21.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern22By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern22By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern22By { - pub fn empty_output_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::EmptyOutputIndex) } + pub fn empty_output_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::EmptyOutputIndex) + } } -pub struct SeriesPattern22 { name: Arc, pub by: SeriesPattern22By } +pub struct SeriesPattern22 { + name: Arc, + pub by: SeriesPattern22By, +} impl SeriesPattern22 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern22By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern22By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern22 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I22 } } -impl SeriesPattern for SeriesPattern22 { fn get(&self, index: Index) -> Option> { _I22.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern22 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I22 + } +} +impl SeriesPattern for SeriesPattern22 { + fn get(&self, index: Index) -> Option> { + _I22.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern23By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern23By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern23By { - pub fn op_return_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::OpReturnIndex) } + pub fn op_return_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::OpReturnIndex) + } } -pub struct SeriesPattern23 { name: Arc, pub by: SeriesPattern23By } +pub struct SeriesPattern23 { + name: Arc, + pub by: SeriesPattern23By, +} impl SeriesPattern23 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern23By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern23By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern23 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I23 } } -impl SeriesPattern for SeriesPattern23 { fn get(&self, index: Index) -> Option> { _I23.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern23 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I23 + } +} +impl SeriesPattern for SeriesPattern23 { + fn get(&self, index: Index) -> Option> { + _I23.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern24By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern24By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern24By { - pub fn p2a_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2AAddrIndex) } + pub fn p2a_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2AAddrIndex) + } } -pub struct SeriesPattern24 { name: Arc, pub by: SeriesPattern24By } +pub struct SeriesPattern24 { + name: Arc, + pub by: SeriesPattern24By, +} impl SeriesPattern24 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern24By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern24By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern24 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I24 } } -impl SeriesPattern for SeriesPattern24 { fn get(&self, index: Index) -> Option> { _I24.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern24 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I24 + } +} +impl SeriesPattern for SeriesPattern24 { + fn get(&self, index: Index) -> Option> { + _I24.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern25By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern25By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern25By { - pub fn p2ms_output_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2MSOutputIndex) } + pub fn p2ms_output_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2MSOutputIndex) + } } -pub struct SeriesPattern25 { name: Arc, pub by: SeriesPattern25By } +pub struct SeriesPattern25 { + name: Arc, + pub by: SeriesPattern25By, +} impl SeriesPattern25 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern25By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern25By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern25 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I25 } } -impl SeriesPattern for SeriesPattern25 { fn get(&self, index: Index) -> Option> { _I25.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern25 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I25 + } +} +impl SeriesPattern for SeriesPattern25 { + fn get(&self, index: Index) -> Option> { + _I25.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern26By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern26By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern26By { - pub fn p2pk33_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2PK33AddrIndex) } + pub fn p2pk33_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2PK33AddrIndex) + } } -pub struct SeriesPattern26 { name: Arc, pub by: SeriesPattern26By } +pub struct SeriesPattern26 { + name: Arc, + pub by: SeriesPattern26By, +} impl SeriesPattern26 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern26By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern26By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern26 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I26 } } -impl SeriesPattern for SeriesPattern26 { fn get(&self, index: Index) -> Option> { _I26.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern26 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I26 + } +} +impl SeriesPattern for SeriesPattern26 { + fn get(&self, index: Index) -> Option> { + _I26.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern27By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern27By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern27By { - pub fn p2pk65_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2PK65AddrIndex) } + pub fn p2pk65_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2PK65AddrIndex) + } } -pub struct SeriesPattern27 { name: Arc, pub by: SeriesPattern27By } +pub struct SeriesPattern27 { + name: Arc, + pub by: SeriesPattern27By, +} impl SeriesPattern27 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern27By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern27By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern27 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I27 } } -impl SeriesPattern for SeriesPattern27 { fn get(&self, index: Index) -> Option> { _I27.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern27 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I27 + } +} +impl SeriesPattern for SeriesPattern27 { + fn get(&self, index: Index) -> Option> { + _I27.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern28By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern28By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern28By { - pub fn p2pkh_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2PKHAddrIndex) } + pub fn p2pkh_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2PKHAddrIndex) + } } -pub struct SeriesPattern28 { name: Arc, pub by: SeriesPattern28By } +pub struct SeriesPattern28 { + name: Arc, + pub by: SeriesPattern28By, +} impl SeriesPattern28 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern28By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern28By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern28 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I28 } } -impl SeriesPattern for SeriesPattern28 { fn get(&self, index: Index) -> Option> { _I28.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern28 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I28 + } +} +impl SeriesPattern for SeriesPattern28 { + fn get(&self, index: Index) -> Option> { + _I28.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern29By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern29By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern29By { - pub fn p2sh_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2SHAddrIndex) } + pub fn p2sh_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2SHAddrIndex) + } } -pub struct SeriesPattern29 { name: Arc, pub by: SeriesPattern29By } +pub struct SeriesPattern29 { + name: Arc, + pub by: SeriesPattern29By, +} impl SeriesPattern29 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern29By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern29By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern29 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I29 } } -impl SeriesPattern for SeriesPattern29 { fn get(&self, index: Index) -> Option> { _I29.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern29 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I29 + } +} +impl SeriesPattern for SeriesPattern29 { + fn get(&self, index: Index) -> Option> { + _I29.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern30By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern30By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern30By { - pub fn p2tr_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2TRAddrIndex) } + pub fn p2tr_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2TRAddrIndex) + } } -pub struct SeriesPattern30 { name: Arc, pub by: SeriesPattern30By } +pub struct SeriesPattern30 { + name: Arc, + pub by: SeriesPattern30By, +} impl SeriesPattern30 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern30By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern30By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern30 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I30 } } -impl SeriesPattern for SeriesPattern30 { fn get(&self, index: Index) -> Option> { _I30.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern30 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I30 + } +} +impl SeriesPattern for SeriesPattern30 { + fn get(&self, index: Index) -> Option> { + _I30.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern31By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern31By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern31By { - pub fn p2wpkh_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2WPKHAddrIndex) } + pub fn p2wpkh_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2WPKHAddrIndex) + } } -pub struct SeriesPattern31 { name: Arc, pub by: SeriesPattern31By } +pub struct SeriesPattern31 { + name: Arc, + pub by: SeriesPattern31By, +} impl SeriesPattern31 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern31By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern31By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern31 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I31 } } -impl SeriesPattern for SeriesPattern31 { fn get(&self, index: Index) -> Option> { _I31.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern31 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I31 + } +} +impl SeriesPattern for SeriesPattern31 { + fn get(&self, index: Index) -> Option> { + _I31.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern32By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern32By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern32By { - pub fn p2wsh_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::P2WSHAddrIndex) } + pub fn p2wsh_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::P2WSHAddrIndex) + } } -pub struct SeriesPattern32 { name: Arc, pub by: SeriesPattern32By } +pub struct SeriesPattern32 { + name: Arc, + pub by: SeriesPattern32By, +} impl SeriesPattern32 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern32By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern32By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern32 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I32 } } -impl SeriesPattern for SeriesPattern32 { fn get(&self, index: Index) -> Option> { _I32.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern32 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I32 + } +} +impl SeriesPattern for SeriesPattern32 { + fn get(&self, index: Index) -> Option> { + _I32.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern33By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern33By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern33By { - pub fn unknown_output_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::UnknownOutputIndex) } + pub fn unknown_output_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::UnknownOutputIndex) + } } -pub struct SeriesPattern33 { name: Arc, pub by: SeriesPattern33By } +pub struct SeriesPattern33 { + name: Arc, + pub by: SeriesPattern33By, +} impl SeriesPattern33 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern33By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern33By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern33 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I33 } } -impl SeriesPattern for SeriesPattern33 { fn get(&self, index: Index) -> Option> { _I33.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern33 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I33 + } +} +impl SeriesPattern for SeriesPattern33 { + fn get(&self, index: Index) -> Option> { + _I33.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern34By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern34By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern34By { - pub fn funded_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::FundedAddrIndex) } + pub fn funded_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::FundedAddrIndex) + } } -pub struct SeriesPattern34 { name: Arc, pub by: SeriesPattern34By } +pub struct SeriesPattern34 { + name: Arc, + pub by: SeriesPattern34By, +} impl SeriesPattern34 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern34By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern34By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern34 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I34 } } -impl SeriesPattern for SeriesPattern34 { fn get(&self, index: Index) -> Option> { _I34.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern34 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I34 + } +} +impl SeriesPattern for SeriesPattern34 { + fn get(&self, index: Index) -> Option> { + _I34.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} -pub struct SeriesPattern35By { client: Arc, name: Arc, _marker: std::marker::PhantomData } +pub struct SeriesPattern35By { + client: Arc, + name: Arc, + _marker: std::marker::PhantomData, +} impl SeriesPattern35By { - pub fn empty_addr_index(&self) -> SeriesEndpoint { _ep(&self.client, &self.name, Index::EmptyAddrIndex) } + pub fn empty_addr_index(&self) -> SeriesEndpoint { + _ep(&self.client, &self.name, Index::EmptyAddrIndex) + } } -pub struct SeriesPattern35 { name: Arc, pub by: SeriesPattern35By } +pub struct SeriesPattern35 { + name: Arc, + pub by: SeriesPattern35By, +} impl SeriesPattern35 { - pub fn new(client: Arc, name: String) -> Self { let name: Arc = name.into(); Self { name: name.clone(), by: SeriesPattern35By { client, name, _marker: std::marker::PhantomData } } } - pub fn name(&self) -> &str { &self.name } + pub fn new(client: Arc, name: String) -> Self { + let name: Arc = name.into(); + Self { + name: name.clone(), + by: SeriesPattern35By { + client, + name, + _marker: std::marker::PhantomData, + }, + } + } + pub fn name(&self) -> &str { + &self.name + } } -impl AnySeriesPattern for SeriesPattern35 { fn name(&self) -> &str { &self.name } fn indexes(&self) -> &'static [Index] { _I35 } } -impl SeriesPattern for SeriesPattern35 { fn get(&self, index: Index) -> Option> { _I35.contains(&index).then(|| _ep(&self.by.client, &self.by.name, index)) } } +impl AnySeriesPattern for SeriesPattern35 { + fn name(&self) -> &str { + &self.name + } + fn indexes(&self) -> &'static [Index] { + _I35 + } +} +impl SeriesPattern for SeriesPattern35 { + fn get(&self, index: Index) -> Option> { + _I35.contains(&index) + .then(|| _ep(&self.by.client, &self.by.name, index)) + } +} // Reusable pattern structs /// Pattern struct for repeated tree structure. -pub struct IndexPct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99ScorePattern { +pub struct IndexPct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99ScorePattern +{ pub index: SeriesPattern1, pub pct0_01: CentsSatsUsdPattern, pub pct0_5: CentsSatsUsdPattern, @@ -1127,7 +2469,8 @@ impl IndexPct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct9 } /// Pattern struct for repeated tree structure. -pub struct Pct05Pct10Pct15Pct20Pct25Pct30Pct35Pct40Pct45Pct50Pct55Pct60Pct65Pct70Pct75Pct80Pct85Pct90Pct95Pattern { +pub struct Pct05Pct10Pct15Pct20Pct25Pct30Pct35Pct40Pct45Pct50Pct55Pct60Pct65Pct70Pct75Pct80Pct85Pct90Pct95Pattern +{ pub pct05: CentsSatsUsdPattern, pub pct10: CentsSatsUsdPattern, pub pct15: CentsSatsUsdPattern, @@ -1248,18 +2591,51 @@ impl AllEmptyOpP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern { pub fn new(client: Arc, acc: String) -> Self { Self { all: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "bis")), - empty: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_empty_outputs_output")), - op_return: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_op_return_output")), + empty: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_empty_outputs_output"), + ), + op_return: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_op_return_output"), + ), p2a: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2a_output")), - p2ms: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2ms_output")), - p2pk33: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2pk33_output")), - p2pk65: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2pk65_output")), - p2pkh: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2pkh_output")), - p2sh: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2sh_output")), - p2tr: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2tr_output")), - p2wpkh: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2wpkh_output")), - p2wsh: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_p2wsh_output")), - unknown: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "with_unknown_outputs_output")), + p2ms: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2ms_output"), + ), + p2pk33: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2pk33_output"), + ), + p2pk65: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2pk65_output"), + ), + p2pkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2pkh_output"), + ), + p2sh: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2sh_output"), + ), + p2tr: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2tr_output"), + ), + p2wpkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2wpkh_output"), + ), + p2wsh: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_p2wsh_output"), + ), + unknown: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "with_unknown_outputs_output"), + ), } } } @@ -1374,15 +2750,27 @@ impl CapCapitalizedGrossLossMvrvNetPeakPriceProfitSellSoprPattern { Self { cap: CentsDeltaToUsdPattern::new(client.clone(), _m(&acc, "realized_cap")), capitalized: PricePattern::new(client.clone(), _m(&acc, "capitalized_price")), - gross_pnl: BlockCumulativeSumPattern::new(client.clone(), _m(&acc, "realized_gross_pnl")), + gross_pnl: BlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "realized_gross_pnl"), + ), loss: BlockCumulativeNegativeSumPattern::new(client.clone(), _m(&acc, "realized_loss")), mvrv: SeriesPattern1::new(client.clone(), _m(&acc, "mvrv")), net_pnl: BlockChangeCumulativeDeltaSumPattern::new(client.clone(), _m(&acc, "net")), - peak_regret: BlockCumulativeSumPattern::new(client.clone(), _m(&acc, "realized_peak_regret")), + peak_regret: BlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "realized_peak_regret"), + ), price: CentsPpmRatioSatsUsdPattern::new(client.clone(), _m(&acc, "realized_price")), profit: BlockCumulativeSumPattern::new(client.clone(), _m(&acc, "realized_profit")), - profit_to_loss_ratio: _1m1w1y24hPattern::new(client.clone(), _m(&acc, "realized_profit_to_loss_ratio")), - sell_side_risk_ratio: _1m1w1y24hPattern8::new(client.clone(), _m(&acc, "sell_side_risk_ratio")), + profit_to_loss_ratio: _1m1w1y24hPattern::new( + client.clone(), + _m(&acc, "realized_profit_to_loss_ratio"), + ), + sell_side_risk_ratio: _1m1w1y24hPattern8::new( + client.clone(), + _m(&acc, "sell_side_risk_ratio"), + ), sopr: AdjustedRatioValuePattern::new(client.clone(), acc.clone()), } } @@ -1408,18 +2796,36 @@ impl EmptyOpP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern2 { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - empty: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "empty_outputs_output")), - op_return: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "op_return_output")), + empty: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "empty_outputs_output"), + ), + op_return: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "op_return_output"), + ), p2a: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2a_output")), p2ms: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2ms_output")), - p2pk33: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2pk33_output")), - p2pk65: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2pk65_output")), + p2pk33: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "p2pk33_output"), + ), + p2pk65: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "p2pk65_output"), + ), p2pkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2pkh_output")), p2sh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2sh_output")), p2tr: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2tr_output")), - p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2wpkh_output")), + p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "p2wpkh_output"), + ), p2wsh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2wsh_output")), - unknown: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "unknown_outputs_output")), + unknown: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "unknown_outputs_output"), + ), } } } @@ -1477,17 +2883,32 @@ impl EmptyP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern2 { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - empty: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "empty_outputs_prevout")), + empty: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "empty_outputs_prevout"), + ), p2a: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2a_prevout")), p2ms: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2ms_prevout")), - p2pk33: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2pk33_prevout")), - p2pk65: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2pk65_prevout")), + p2pk33: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "p2pk33_prevout"), + ), + p2pk65: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "p2pk65_prevout"), + ), p2pkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2pkh_prevout")), p2sh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2sh_prevout")), p2tr: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2tr_prevout")), - p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2wpkh_prevout")), + p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "p2wpkh_prevout"), + ), p2wsh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "p2wsh_prevout")), - unknown: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "unknown_outputs_prevout")), + unknown: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + _m(&acc, "unknown_outputs_prevout"), + ), } } } @@ -1707,8 +3128,14 @@ impl CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2 { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - capitalized_cap_in_loss_raw: SeriesPattern18::new(client.clone(), _m(&acc, "capitalized_cap_in_loss_raw")), - capitalized_cap_in_profit_raw: SeriesPattern18::new(client.clone(), _m(&acc, "capitalized_cap_in_profit_raw")), + capitalized_cap_in_loss_raw: SeriesPattern18::new( + client.clone(), + _m(&acc, "capitalized_cap_in_loss_raw"), + ), + capitalized_cap_in_profit_raw: SeriesPattern18::new( + client.clone(), + _m(&acc, "capitalized_cap_in_profit_raw"), + ), gross_pnl: CentsUsdPattern3::new(client.clone(), _m(&acc, "unrealized_gross_pnl")), invested_capital: InPattern2::new(client.clone(), _m(&acc, "invested_capital_in")), loss: CentsNegativeToUsdPattern2::new(client.clone(), _m(&acc, "unrealized_loss")), @@ -1767,11 +3194,17 @@ impl ChainDataFeeFeesOutputTxPattern { pub fn new(client: Arc, acc: String) -> Self { Self { chain_share: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "chain_share")), - data_bytes: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "data_bytes")), + data_bytes: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "data_bytes"), + ), data_share: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "data_share")), fee_share: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), _m(&acc, "fee_share")), fees: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "fees")), - output_count: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "output_count")), + output_count: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "output_count"), + ), tx_count: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "tx_count")), tx_vsize: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "tx_vsize")), } @@ -1859,7 +3292,10 @@ impl CapLossMvrvNetPriceProfitSoprPattern { cap: CentsDeltaUsdPattern::new(client.clone(), _m(&acc, "realized_cap")), loss: BlockCumulativeNegativeSumPattern::new(client.clone(), _m(&acc, "realized_loss")), mvrv: SeriesPattern1::new(client.clone(), _m(&acc, "mvrv")), - net_pnl: BlockCumulativeDeltaSumPattern::new(client.clone(), _m(&acc, "net_realized_pnl")), + net_pnl: BlockCumulativeDeltaSumPattern::new( + client.clone(), + _m(&acc, "net_realized_pnl"), + ), price: CentsPpmRatioSatsUsdPattern::new(client.clone(), _m(&acc, "realized_price")), profit: BlockCumulativeSumPattern::new(client.clone(), _m(&acc, "realized_profit")), sopr: RatioValuePattern::new(client.clone(), acc.clone()), @@ -1882,9 +3318,18 @@ impl CoindaysLivelinessRatioSupplyVaultednessPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - coindays_consumed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_consumed")), - coindays_created: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_created")), - coindays_stored: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_stored")), + coindays_consumed: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "coindays_consumed"), + ), + coindays_created: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "coindays_created"), + ), + coindays_stored: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "coindays_stored"), + ), liveliness: SeriesPattern1::new(client.clone(), _m(&acc, "liveliness")), ratio: SeriesPattern1::new(client.clone(), _m(&acc, "activity_to_vaultedness")), supply: ActiveVaultedPattern::new(client.clone(), acc.clone()), @@ -1989,7 +3434,10 @@ impl AverageBlockCumulativeInSumPattern { block: BtcCentsSatsUsdPattern3::new(client.clone(), acc.clone()), cumulative: BtcCentsSatsUsdPattern::new(client.clone(), _m(&acc, "cumulative")), in_loss: AverageBlockCumulativeSumPattern2::new(client.clone(), _m(&acc, "in_loss")), - in_profit: AverageBlockCumulativeSumPattern2::new(client.clone(), _m(&acc, "in_profit")), + in_profit: AverageBlockCumulativeSumPattern2::new( + client.clone(), + _m(&acc, "in_profit"), + ), sum: _1m1w1y24hPattern4::new(client.clone(), _m(&acc, "sum")), } } @@ -2012,7 +3460,10 @@ impl CentsNegativeToUsdPattern2 { cents: SeriesPattern1::new(client.clone(), _m(&acc, "cents")), negative: SeriesPattern1::new(client.clone(), _m(&acc, "neg")), to_mcap: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "to_mcap")), - to_own_gross_pnl: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "to_own_gross_pnl")), + to_own_gross_pnl: PercentPpmRatioPattern2::new( + client.clone(), + _m(&acc, "to_own_gross_pnl"), + ), to_own_mcap: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "to_own_mcap")), usd: SeriesPattern1::new(client.clone(), acc.clone()), } @@ -2125,7 +3576,10 @@ impl ActiveBidirectionalReactivatedReceivingSendingPattern { pub fn new(client: Arc, acc: String) -> Self { Self { active: _1m1w1y24hBlockPattern::new(client.clone(), _m(&acc, "active_addrs")), - bidirectional: _1m1w1y24hBlockPattern::new(client.clone(), _m(&acc, "bidirectional_addrs")), + bidirectional: _1m1w1y24hBlockPattern::new( + client.clone(), + _m(&acc, "bidirectional_addrs"), + ), reactivated: _1m1w1y24hBlockPattern::new(client.clone(), _m(&acc, "reactivated_addrs")), receiving: _1m1w1y24hBlockPattern::new(client.clone(), _m(&acc, "receiving_addrs")), sending: _1m1w1y24hBlockPattern::new(client.clone(), _m(&acc, "sending_addrs")), @@ -2355,7 +3809,10 @@ impl CentsToUsdPattern4 { Self { cents: SeriesPattern1::new(client.clone(), _m(&acc, "cents")), to_mcap: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "to_mcap")), - to_own_gross_pnl: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "to_own_gross_pnl")), + to_own_gross_pnl: PercentPpmRatioPattern2::new( + client.clone(), + _m(&acc, "to_own_gross_pnl"), + ), to_own_mcap: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, "to_own_mcap")), usd: SeriesPattern1::new(client.clone(), acc.clone()), } @@ -2750,7 +4207,10 @@ impl CentsToUsdPattern3 { pub fn new(client: Arc, acc: String) -> Self { Self { cents: SeriesPattern1::new(client.clone(), _m(&acc, "cents")), - to_own_gross_pnl: PercentPpmRatioPattern3::new(client.clone(), _m(&acc, "to_own_gross_pnl")), + to_own_gross_pnl: PercentPpmRatioPattern3::new( + client.clone(), + _m(&acc, "to_own_gross_pnl"), + ), to_own_mcap: PercentPpmRatioPattern3::new(client.clone(), _m(&acc, "to_own_mcap")), usd: SeriesPattern1::new(client.clone(), acc.clone()), } @@ -2769,10 +4229,19 @@ impl CoindaysCoinyearsDormancyTransferPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - coindays_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_destroyed")), - coinyears_destroyed: SeriesPattern1::new(client.clone(), _m(&acc, "coinyears_destroyed")), + coindays_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "coindays_destroyed"), + ), + coinyears_destroyed: SeriesPattern1::new( + client.clone(), + _m(&acc, "coinyears_destroyed"), + ), dormancy: _1m1w1y24hPattern::new(client.clone(), _m(&acc, "dormancy")), - transfer_volume: AverageBlockCumulativeInSumPattern::new(client.clone(), _m(&acc, "transfer_volume")), + transfer_volume: AverageBlockCumulativeInSumPattern::new( + client.clone(), + _m(&acc, "transfer_volume"), + ), } } } @@ -2830,9 +4299,17 @@ impl NuplRealizedSupplyUnrealizedPattern { pub fn new(client: Arc, acc: String) -> Self { Self { nupl: PpmRatioPattern::new(client.clone(), _m(&acc, "nupl")), - realized_cap: AllSthPattern::new(client.clone(), acc.clone(), "realized_cap".to_string()), + realized_cap: AllSthPattern::new( + client.clone(), + acc.clone(), + "realized_cap".to_string(), + ), supply: AllSthPattern2::new(client.clone(), acc.clone()), - unrealized_pnl: AllSthPattern::new(client.clone(), acc.clone(), "unrealized_pnl".to_string()), + unrealized_pnl: AllSthPattern::new( + client.clone(), + acc.clone(), + "unrealized_pnl".to_string(), + ), } } } @@ -2890,7 +4367,10 @@ impl AdjustedRatioValuePattern { Self { adjusted: RatioTransferValuePattern::new(client.clone(), acc.clone()), ratio: _1m1w1y24hPattern::new(client.clone(), _m(&acc, "sopr")), - value_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "value_destroyed")), + value_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "value_destroyed"), + ), } } } @@ -3058,7 +4538,10 @@ impl CumulativeRollingSumPattern { pub fn new(client: Arc, acc: String) -> Self { Self { cumulative: SeriesPattern1::new(client.clone(), _m(&acc, "cumulative")), - rolling: AverageMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new(client.clone(), acc.clone()), + rolling: AverageMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new( + client.clone(), + acc.clone(), + ), sum: SeriesPattern18::new(client.clone(), _m(&acc, "sum")), } } @@ -3094,8 +4577,14 @@ impl FloorLevelLossPattern { pub fn new(client: Arc, acc: String) -> Self { Self { floor: Pct95Pct98Pct99Pattern::new(client.clone(), _m(&acc, "floor")), - level: Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern::new(client.clone(), _m(&acc, "level")), - loss_threshold: Pct95Pct98Pct99Pattern2::new(client.clone(), _m(&acc, "loss_threshold")), + level: Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern::new( + client.clone(), + _m(&acc, "level"), + ), + loss_threshold: Pct95Pct98Pct99Pattern2::new( + client.clone(), + _m(&acc, "loss_threshold"), + ), } } } @@ -3219,9 +4708,15 @@ impl PpmPriceRatioPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String, disc: String) -> Self { Self { - ppm: SeriesPattern1::new(client.clone(), _m(&acc, &format!("ratio_{disc}_ppm", disc=disc))), + ppm: SeriesPattern1::new( + client.clone(), + _m(&acc, &format!("ratio_{disc}_ppm", disc = disc)), + ), price: CentsSatsUsdPattern::new(client.clone(), _m(&acc, &disc)), - ratio: SeriesPattern1::new(client.clone(), _m(&acc, &format!("ratio_{disc}", disc=disc))), + ratio: SeriesPattern1::new( + client.clone(), + _m(&acc, &format!("ratio_{disc}", disc = disc)), + ), } } } @@ -3238,8 +4733,14 @@ impl RatioTransferValuePattern { pub fn new(client: Arc, acc: String) -> Self { Self { ratio: _1m1w1y24hPattern::new(client.clone(), _m(&acc, "asopr")), - transfer_volume: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "adj_value_created")), - value_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "adj_value_destroyed")), + transfer_volume: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "adj_value_created"), + ), + value_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "adj_value_destroyed"), + ), } } } @@ -3256,8 +4757,14 @@ impl RsiStochPattern { pub fn new(client: Arc, acc: String, disc: String) -> Self { Self { rsi: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, &disc)), - stoch_rsi_d: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, &format!("stoch_d_{disc}", disc=disc))), - stoch_rsi_k: PercentPpmRatioPattern2::new(client.clone(), _m(&acc, &format!("stoch_k_{disc}", disc=disc))), + stoch_rsi_d: PercentPpmRatioPattern2::new( + client.clone(), + _m(&acc, &format!("stoch_d_{disc}", disc = disc)), + ), + stoch_rsi_k: PercentPpmRatioPattern2::new( + client.clone(), + _m(&acc, &format!("stoch_k_{disc}", disc = disc)), + ), } } } @@ -3387,7 +4894,10 @@ impl AllSthPattern { pub fn new(client: Arc, acc: String, disc: String) -> Self { Self { all: SeriesPattern1::new(client.clone(), _m(&acc, &disc)), - sth: SeriesPattern1::new(client.clone(), _m(&acc, &format!("sth_{disc}", disc=disc))), + sth: SeriesPattern1::new( + client.clone(), + _m(&acc, &format!("sth_{disc}", disc = disc)), + ), } } } @@ -3546,8 +5056,14 @@ impl CoindaysTransferPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - coindays_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_destroyed")), - transfer_volume: AverageBlockCumulativeInSumPattern::new(client.clone(), _m(&acc, "transfer_volume")), + coindays_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "coindays_destroyed"), + ), + transfer_volume: AverageBlockCumulativeInSumPattern::new( + client.clone(), + _m(&acc, "transfer_volume"), + ), } } } @@ -3562,8 +5078,14 @@ impl FundedTotalPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - funded: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new(client.clone(), acc.clone()), - total: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new(client.clone(), _p("total", &acc)), + funded: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new( + client.clone(), + acc.clone(), + ), + total: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new( + client.clone(), + _p("total", &acc), + ), } } } @@ -3691,7 +5213,10 @@ impl RatioValuePattern { pub fn new(client: Arc, acc: String) -> Self { Self { ratio: _24hPattern::new(client.clone(), _m(&acc, "sopr_24h")), - value_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "value_destroyed")), + value_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "value_destroyed"), + ), } } } @@ -3712,7 +5237,10 @@ impl SpentUnspentPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - spent_count: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "spent_utxo_count")), + spent_count: AverageBlockCumulativeSumPattern::new( + client.clone(), + _m(&acc, "spent_utxo_count"), + ), unspent_count: BaseDeltaPattern::new(client.clone(), _m(&acc, "utxo_count")), } } @@ -3890,7 +5418,10 @@ impl SeriesTree { pub fn new(client: Arc, base_path: String) -> Self { Self { blocks: SeriesTree_Blocks::new(client.clone(), format!("{base_path}_blocks")), - transactions: SeriesTree_Transactions::new(client.clone(), format!("{base_path}_transactions")), + transactions: SeriesTree_Transactions::new( + client.clone(), + format!("{base_path}_transactions"), + ), inputs: SeriesTree_Inputs::new(client.clone(), format!("{base_path}_inputs")), outputs: SeriesTree_Outputs::new(client.clone(), format!("{base_path}_outputs")), addrs: SeriesTree_Addrs::new(client.clone(), format!("{base_path}_addrs")), @@ -3902,7 +5433,10 @@ impl SeriesTree { bedrock: SeriesTree_Bedrock::new(client.clone(), format!("{base_path}_bedrock")), constants: SeriesTree_Constants::new(client.clone(), format!("{base_path}_constants")), indexes: SeriesTree_Indexes::new(client.clone(), format!("{base_path}_indexes")), - indicators: SeriesTree_Indicators::new(client.clone(), format!("{base_path}_indicators")), + indicators: SeriesTree_Indicators::new( + client.clone(), + format!("{base_path}_indicators"), + ), investing: SeriesTree_Investing::new(client.clone(), format!("{base_path}_investing")), market: SeriesTree_Market::new(client.clone(), format!("{base_path}_market")), pools: SeriesTree_Pools::new(client.clone(), format!("{base_path}_pools")), @@ -3937,18 +5471,36 @@ impl SeriesTree_Blocks { Self { blockhash: SeriesPattern18::new(client.clone(), "blockhash".to_string()), coinbase_tag: SeriesPattern18::new(client.clone(), "coinbase_tag".to_string()), - difficulty: SeriesTree_Blocks_Difficulty::new(client.clone(), format!("{base_path}_difficulty")), + difficulty: SeriesTree_Blocks_Difficulty::new( + client.clone(), + format!("{base_path}_difficulty"), + ), time: SeriesTree_Blocks_Time::new(client.clone(), format!("{base_path}_time")), size: SeriesTree_Blocks_Size::new(client.clone(), format!("{base_path}_size")), - weight: AverageBaseCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new(client.clone(), "block_weight".to_string()), + weight: AverageBaseCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new( + client.clone(), + "block_weight".to_string(), + ), segwit_txs: SeriesPattern18::new(client.clone(), "segwit_txs".to_string()), segwit_size: SeriesPattern18::new(client.clone(), "segwit_size".to_string()), segwit_weight: SeriesPattern18::new(client.clone(), "segwit_weight".to_string()), count: SeriesTree_Blocks_Count::new(client.clone(), format!("{base_path}_count")), - lookback: SeriesTree_Blocks_Lookback::new(client.clone(), format!("{base_path}_lookback")), - interval: SeriesTree_Blocks_Interval::new(client.clone(), format!("{base_path}_interval")), - vbytes: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new(client.clone(), "block_vbytes".to_string()), - fullness: SeriesTree_Blocks_Fullness::new(client.clone(), format!("{base_path}_fullness")), + lookback: SeriesTree_Blocks_Lookback::new( + client.clone(), + format!("{base_path}_lookback"), + ), + interval: SeriesTree_Blocks_Interval::new( + client.clone(), + format!("{base_path}_interval"), + ), + vbytes: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new( + client.clone(), + "block_vbytes".to_string(), + ), + fullness: SeriesTree_Blocks_Fullness::new( + client.clone(), + format!("{base_path}_fullness"), + ), halving: SeriesTree_Blocks_Halving::new(client.clone(), format!("{base_path}_halving")), } } @@ -3969,9 +5521,15 @@ impl SeriesTree_Blocks_Difficulty { Self { value: SeriesPattern1::new(client.clone(), "difficulty".to_string()), hashrate: SeriesPattern1::new(client.clone(), "difficulty_hashrate".to_string()), - adjustment: PercentPpmRatioPattern3::new(client.clone(), "difficulty_adjustment".to_string()), + adjustment: PercentPpmRatioPattern3::new( + client.clone(), + "difficulty_adjustment".to_string(), + ), epoch: SeriesPattern1::new(client.clone(), "difficulty_epoch".to_string()), - blocks_to_retarget: SeriesPattern1::new(client.clone(), "blocks_to_retarget".to_string()), + blocks_to_retarget: SeriesPattern1::new( + client.clone(), + "blocks_to_retarget".to_string(), + ), days_to_retarget: SeriesPattern1::new(client.clone(), "days_to_retarget".to_string()), } } @@ -4207,14 +5765,29 @@ impl SeriesTree_Transactions { pub fn new(client: Arc, base_path: String) -> Self { Self { raw: SeriesTree_Transactions_Raw::new(client.clone(), format!("{base_path}_raw")), - features: SeriesTree_Transactions_Features::new(client.clone(), format!("{base_path}_features")), + features: SeriesTree_Transactions_Features::new( + client.clone(), + format!("{base_path}_features"), + ), count: SeriesTree_Transactions_Count::new(client.clone(), format!("{base_path}_count")), size: SeriesTree_Transactions_Size::new(client.clone(), format!("{base_path}_size")), fees: SeriesTree_Transactions_Fees::new(client.clone(), format!("{base_path}_fees")), - patterns: SeriesTree_Transactions_Patterns::new(client.clone(), format!("{base_path}_patterns")), - policy: SeriesTree_Transactions_Policy::new(client.clone(), format!("{base_path}_policy")), - versions: SeriesTree_Transactions_Versions::new(client.clone(), format!("{base_path}_versions")), - volume: SeriesTree_Transactions_Volume::new(client.clone(), format!("{base_path}_volume")), + patterns: SeriesTree_Transactions_Patterns::new( + client.clone(), + format!("{base_path}_patterns"), + ), + policy: SeriesTree_Transactions_Policy::new( + client.clone(), + format!("{base_path}_policy"), + ), + versions: SeriesTree_Transactions_Versions::new( + client.clone(), + format!("{base_path}_versions"), + ), + volume: SeriesTree_Transactions_Volume::new( + client.clone(), + format!("{base_path}_volume"), + ), } } } @@ -4243,9 +5816,15 @@ impl SeriesTree_Transactions_Raw { weight: SeriesPattern19::new(client.clone(), "tx_weight".to_string()), total_size: SeriesPattern19::new(client.clone(), "total_size".to_string()), total_sigop_cost: SeriesPattern19::new(client.clone(), "total_sigop_cost".to_string()), - is_explicitly_rbf: SeriesPattern19::new(client.clone(), "is_explicitly_rbf".to_string()), + is_explicitly_rbf: SeriesPattern19::new( + client.clone(), + "is_explicitly_rbf".to_string(), + ), first_txin_index: SeriesPattern19::new(client.clone(), "first_txin_index".to_string()), - first_txout_index: SeriesPattern19::new(client.clone(), "first_txout_index".to_string()), + first_txout_index: SeriesPattern19::new( + client.clone(), + "first_txout_index".to_string(), + ), } } } @@ -4279,7 +5858,10 @@ pub struct SeriesTree_Transactions_Features { impl SeriesTree_Transactions_Features { pub fn new(client: Arc, base_path: String) -> Self { Self { - count: SeriesTree_Transactions_Features_Count::new(client.clone(), format!("{base_path}_count")), + count: SeriesTree_Transactions_Features_Count::new( + client.clone(), + format!("{base_path}_count"), + ), has_p2pk: SeriesPattern19::new(client.clone(), "has_p2pk".to_string()), has_p2ms: SeriesPattern19::new(client.clone(), "has_p2ms".to_string()), has_p2pkh: SeriesPattern19::new(client.clone(), "has_p2pkh".to_string()), @@ -4292,14 +5874,26 @@ impl SeriesTree_Transactions_Features { has_empty: SeriesPattern19::new(client.clone(), "has_empty".to_string()), has_unknown: SeriesPattern19::new(client.clone(), "has_unknown".to_string()), has_fake_pubkey: SeriesPattern19::new(client.clone(), "has_fake_pubkey".to_string()), - has_fake_scripthash: SeriesPattern19::new(client.clone(), "has_fake_scripthash".to_string()), + has_fake_scripthash: SeriesPattern19::new( + client.clone(), + "has_fake_scripthash".to_string(), + ), has_inscription: SeriesPattern19::new(client.clone(), "has_inscription".to_string()), has_annex: SeriesPattern19::new(client.clone(), "has_annex".to_string()), has_sighash_all: SeriesPattern19::new(client.clone(), "has_sighash_all".to_string()), has_sighash_none: SeriesPattern19::new(client.clone(), "has_sighash_none".to_string()), - has_sighash_single: SeriesPattern19::new(client.clone(), "has_sighash_single".to_string()), - has_sighash_default: SeriesPattern19::new(client.clone(), "has_sighash_default".to_string()), - has_sighash_anyone_can_pay: SeriesPattern19::new(client.clone(), "has_sighash_anyone_can_pay".to_string()), + has_sighash_single: SeriesPattern19::new( + client.clone(), + "has_sighash_single".to_string(), + ), + has_sighash_default: SeriesPattern19::new( + client.clone(), + "has_sighash_default".to_string(), + ), + has_sighash_anyone_can_pay: SeriesPattern19::new( + client.clone(), + "has_sighash_anyone_can_pay".to_string(), + ), has_dust_output: SeriesPattern19::new(client.clone(), "has_dust_output".to_string()), } } @@ -4343,8 +5937,14 @@ impl SeriesTree_Transactions_Features_Count { v1: SeriesPattern18::new(client.clone(), "tx_count_v1".to_string()), v2: SeriesPattern18::new(client.clone(), "tx_count_v2".to_string()), v3: SeriesPattern18::new(client.clone(), "tx_count_v3".to_string()), - other_version: SeriesPattern18::new(client.clone(), "tx_count_other_version".to_string()), - explicitly_rbf: SeriesPattern18::new(client.clone(), "tx_count_explicitly_rbf".to_string()), + other_version: SeriesPattern18::new( + client.clone(), + "tx_count_other_version".to_string(), + ), + explicitly_rbf: SeriesPattern18::new( + client.clone(), + "tx_count_explicitly_rbf".to_string(), + ), one_input: SeriesPattern18::new(client.clone(), "tx_count_one_input".to_string()), one_output: SeriesPattern18::new(client.clone(), "tx_count_one_output".to_string()), p2pk: SeriesPattern18::new(client.clone(), "tx_count_p2pk".to_string()), @@ -4359,14 +5959,26 @@ impl SeriesTree_Transactions_Features_Count { empty: SeriesPattern18::new(client.clone(), "tx_count_empty".to_string()), unknown: SeriesPattern18::new(client.clone(), "tx_count_unknown".to_string()), fake_pubkey: SeriesPattern18::new(client.clone(), "tx_count_fake_pubkey".to_string()), - fake_scripthash: SeriesPattern18::new(client.clone(), "tx_count_fake_scripthash".to_string()), + fake_scripthash: SeriesPattern18::new( + client.clone(), + "tx_count_fake_scripthash".to_string(), + ), inscription: SeriesPattern18::new(client.clone(), "tx_count_inscription".to_string()), annex: SeriesPattern18::new(client.clone(), "tx_count_annex".to_string()), sighash_all: SeriesPattern18::new(client.clone(), "tx_count_sighash_all".to_string()), sighash_none: SeriesPattern18::new(client.clone(), "tx_count_sighash_none".to_string()), - sighash_single: SeriesPattern18::new(client.clone(), "tx_count_sighash_single".to_string()), - sighash_default: SeriesPattern18::new(client.clone(), "tx_count_sighash_default".to_string()), - sighash_anyone_can_pay: SeriesPattern18::new(client.clone(), "tx_count_sighash_anyone_can_pay".to_string()), + sighash_single: SeriesPattern18::new( + client.clone(), + "tx_count_sighash_single".to_string(), + ), + sighash_default: SeriesPattern18::new( + client.clone(), + "tx_count_sighash_default".to_string(), + ), + sighash_anyone_can_pay: SeriesPattern18::new( + client.clone(), + "tx_count_sighash_anyone_can_pay".to_string(), + ), dust_output: SeriesPattern18::new(client.clone(), "tx_count_dust_output".to_string()), } } @@ -4380,7 +5992,10 @@ pub struct SeriesTree_Transactions_Count { impl SeriesTree_Transactions_Count { pub fn new(client: Arc, base_path: String) -> Self { Self { - total: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new(client.clone(), "tx_count".to_string()), + total: AverageBlockCumulativeMaxMedianMinPct10Pct25Pct75Pct90SumPattern::new( + client.clone(), + "tx_count".to_string(), + ), } } } @@ -4394,8 +6009,14 @@ pub struct SeriesTree_Transactions_Size { impl SeriesTree_Transactions_Size { pub fn new(client: Arc, base_path: String) -> Self { Self { - vsize: SeriesTree_Transactions_Size_Vsize::new(client.clone(), format!("{base_path}_vsize")), - weight: SeriesTree_Transactions_Size_Weight::new(client.clone(), format!("{base_path}_weight")), + vsize: SeriesTree_Transactions_Size_Vsize::new( + client.clone(), + format!("{base_path}_vsize"), + ), + weight: SeriesTree_Transactions_Size_Weight::new( + client.clone(), + format!("{base_path}_weight"), + ), } } } @@ -4411,8 +6032,14 @@ impl SeriesTree_Transactions_Size_Vsize { pub fn new(client: Arc, base_path: String) -> Self { Self { tx_index: SeriesPattern19::new(client.clone(), "tx_vsize".to_string()), - block: MaxMedianMinPct10Pct25Pct75Pct90Pattern2::new(client.clone(), "tx_vsize".to_string()), - _6b: MaxMedianMinPct10Pct25Pct75Pct90Pattern2::new(client.clone(), "tx_vsize_6b".to_string()), + block: MaxMedianMinPct10Pct25Pct75Pct90Pattern2::new( + client.clone(), + "tx_vsize".to_string(), + ), + _6b: MaxMedianMinPct10Pct25Pct75Pct90Pattern2::new( + client.clone(), + "tx_vsize_6b".to_string(), + ), } } } @@ -4426,8 +6053,14 @@ pub struct SeriesTree_Transactions_Size_Weight { impl SeriesTree_Transactions_Size_Weight { pub fn new(client: Arc, base_path: String) -> Self { Self { - block: MaxMedianMinPct10Pct25Pct75Pct90Pattern::new(client.clone(), "tx_weight".to_string()), - _6b: MaxMedianMinPct10Pct25Pct75Pct90Pattern::new(client.clone(), "tx_weight_6b".to_string()), + block: MaxMedianMinPct10Pct25Pct75Pct90Pattern::new( + client.clone(), + "tx_weight".to_string(), + ), + _6b: MaxMedianMinPct10Pct25Pct75Pct90Pattern::new( + client.clone(), + "tx_weight_6b".to_string(), + ), } } } @@ -4447,12 +6080,18 @@ pub struct SeriesTree_Transactions_Fees { impl SeriesTree_Transactions_Fees { pub fn new(client: Arc, base_path: String) -> Self { Self { - count: SeriesTree_Transactions_Fees_Count::new(client.clone(), format!("{base_path}_count")), + count: SeriesTree_Transactions_Fees_Count::new( + client.clone(), + format!("{base_path}_count"), + ), input_value: SeriesPattern19::new(client.clone(), "input_value".to_string()), output_value: SeriesPattern19::new(client.clone(), "output_value".to_string()), fee: _6bBlockTxPattern::new(client.clone(), "fee".to_string()), fee_rate: SeriesPattern19::new(client.clone(), "fee_rate".to_string()), - effective_fee_rate: _6bBlockTxPattern::new(client.clone(), "effective_fee_rate".to_string()), + effective_fee_rate: _6bBlockTxPattern::new( + client.clone(), + "effective_fee_rate".to_string(), + ), is_cpfp_parent: SeriesPattern19::new(client.clone(), "is_cpfp_parent".to_string()), is_cpfp_child: SeriesPattern19::new(client.clone(), "is_cpfp_child".to_string()), } @@ -4485,7 +6124,10 @@ pub struct SeriesTree_Transactions_Patterns { impl SeriesTree_Transactions_Patterns { pub fn new(client: Arc, base_path: String) -> Self { Self { - count: SeriesTree_Transactions_Patterns_Count::new(client.clone(), format!("{base_path}_count")), + count: SeriesTree_Transactions_Patterns_Count::new( + client.clone(), + format!("{base_path}_count"), + ), is_coinjoin: SeriesPattern19::new(client.clone(), "is_coinjoin".to_string()), is_consolidation: SeriesPattern19::new(client.clone(), "is_consolidation".to_string()), is_batch_payout: SeriesPattern19::new(client.clone(), "is_batch_payout".to_string()), @@ -4539,7 +6181,10 @@ impl SeriesTree_Transactions_Versions { v1: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_v1".to_string()), v2: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_v2".to_string()), v3: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_v3".to_string()), - other: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_other_version".to_string()), + other: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_other_version".to_string(), + ), } } } @@ -4553,7 +6198,10 @@ pub struct SeriesTree_Transactions_Volume { impl SeriesTree_Transactions_Volume { pub fn new(client: Arc, base_path: String) -> Self { Self { - transfer_volume: AverageBlockCumulativeSumPattern2::new(client.clone(), "transfer_volume_bis".to_string()), + transfer_volume: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "transfer_volume_bis".to_string(), + ), tx_per_sec: _1m1w1y24hPattern::new(client.clone(), "tx_per_sec".to_string()), } } @@ -4627,10 +6275,22 @@ pub struct SeriesTree_Inputs_ByType { impl SeriesTree_Inputs_ByType { pub fn new(client: Arc, base_path: String) -> Self { Self { - input_count: SeriesTree_Inputs_ByType_InputCount::new(client.clone(), format!("{base_path}_input_count")), - input_share: SeriesTree_Inputs_ByType_InputShare::new(client.clone(), format!("{base_path}_input_share")), - tx_count: SeriesTree_Inputs_ByType_TxCount::new(client.clone(), format!("{base_path}_tx_count")), - tx_share: EmptyP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern2::new(client.clone(), "tx_share_with".to_string()), + input_count: SeriesTree_Inputs_ByType_InputCount::new( + client.clone(), + format!("{base_path}_input_count"), + ), + input_share: SeriesTree_Inputs_ByType_InputShare::new( + client.clone(), + format!("{base_path}_input_share"), + ), + tx_count: SeriesTree_Inputs_ByType_TxCount::new( + client.clone(), + format!("{base_path}_tx_count"), + ), + tx_share: EmptyP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern2::new( + client.clone(), + "tx_share_with".to_string(), + ), } } } @@ -4654,18 +6314,54 @@ pub struct SeriesTree_Inputs_ByType_InputCount { impl SeriesTree_Inputs_ByType_InputCount { pub fn new(client: Arc, base_path: String) -> Self { Self { - all: AverageBlockCumulativeSumPattern::new(client.clone(), "input_count_bis".to_string()), - p2pk65: AverageBlockCumulativeSumPattern::new(client.clone(), "p2pk65_prevout_count".to_string()), - p2pk33: AverageBlockCumulativeSumPattern::new(client.clone(), "p2pk33_prevout_count".to_string()), - p2pkh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2pkh_prevout_count".to_string()), - p2ms: AverageBlockCumulativeSumPattern::new(client.clone(), "p2ms_prevout_count".to_string()), - p2sh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2sh_prevout_count".to_string()), - p2wpkh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2wpkh_prevout_count".to_string()), - p2wsh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2wsh_prevout_count".to_string()), - p2tr: AverageBlockCumulativeSumPattern::new(client.clone(), "p2tr_prevout_count".to_string()), - p2a: AverageBlockCumulativeSumPattern::new(client.clone(), "p2a_prevout_count".to_string()), - unknown: AverageBlockCumulativeSumPattern::new(client.clone(), "unknown_outputs_prevout_count".to_string()), - empty: AverageBlockCumulativeSumPattern::new(client.clone(), "empty_outputs_prevout_count".to_string()), + all: AverageBlockCumulativeSumPattern::new( + client.clone(), + "input_count_bis".to_string(), + ), + p2pk65: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2pk65_prevout_count".to_string(), + ), + p2pk33: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2pk33_prevout_count".to_string(), + ), + p2pkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2pkh_prevout_count".to_string(), + ), + p2ms: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2ms_prevout_count".to_string(), + ), + p2sh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2sh_prevout_count".to_string(), + ), + p2wpkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2wpkh_prevout_count".to_string(), + ), + p2wsh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2wsh_prevout_count".to_string(), + ), + p2tr: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2tr_prevout_count".to_string(), + ), + p2a: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2a_prevout_count".to_string(), + ), + unknown: AverageBlockCumulativeSumPattern::new( + client.clone(), + "unknown_outputs_prevout_count".to_string(), + ), + empty: AverageBlockCumulativeSumPattern::new( + client.clone(), + "empty_outputs_prevout_count".to_string(), + ), } } } @@ -4688,17 +6384,50 @@ pub struct SeriesTree_Inputs_ByType_InputShare { impl SeriesTree_Inputs_ByType_InputShare { pub fn new(client: Arc, base_path: String) -> Self { Self { - p2pk65: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2pk65_prevout_share".to_string()), - p2pk33: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2pk33_prevout_share".to_string()), - p2pkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2pkh_prevout_share".to_string()), - p2ms: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2ms_prevout_share".to_string()), - p2sh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2sh_prevout_share".to_string()), - p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2wpkh_prevout_share".to_string()), - p2wsh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2wsh_prevout_share".to_string()), - p2tr: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2tr_prevout_share".to_string()), - p2a: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2a_prevout_share".to_string()), - unknown: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "unknown_outputs_prevout_share".to_string()), - empty: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "empty_outputs_prevout_share".to_string()), + p2pk65: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2pk65_prevout_share".to_string(), + ), + p2pk33: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2pk33_prevout_share".to_string(), + ), + p2pkh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2pkh_prevout_share".to_string(), + ), + p2ms: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2ms_prevout_share".to_string(), + ), + p2sh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2sh_prevout_share".to_string(), + ), + p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2wpkh_prevout_share".to_string(), + ), + p2wsh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2wsh_prevout_share".to_string(), + ), + p2tr: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2tr_prevout_share".to_string(), + ), + p2a: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2a_prevout_share".to_string(), + ), + unknown: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "unknown_outputs_prevout_share".to_string(), + ), + empty: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "empty_outputs_prevout_share".to_string(), + ), } } } @@ -4722,18 +6451,54 @@ pub struct SeriesTree_Inputs_ByType_TxCount { impl SeriesTree_Inputs_ByType_TxCount { pub fn new(client: Arc, base_path: String) -> Self { Self { - all: AverageBlockCumulativeSumPattern::new(client.clone(), "non_coinbase_tx_count".to_string()), - p2pk65: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2pk65_prevout".to_string()), - p2pk33: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2pk33_prevout".to_string()), - p2pkh: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2pkh_prevout".to_string()), - p2ms: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2ms_prevout".to_string()), - p2sh: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2sh_prevout".to_string()), - p2wpkh: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2wpkh_prevout".to_string()), - p2wsh: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2wsh_prevout".to_string()), - p2tr: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2tr_prevout".to_string()), - p2a: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_p2a_prevout".to_string()), - unknown: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_unknown_outputs_prevout".to_string()), - empty: AverageBlockCumulativeSumPattern::new(client.clone(), "tx_count_with_empty_outputs_prevout".to_string()), + all: AverageBlockCumulativeSumPattern::new( + client.clone(), + "non_coinbase_tx_count".to_string(), + ), + p2pk65: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2pk65_prevout".to_string(), + ), + p2pk33: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2pk33_prevout".to_string(), + ), + p2pkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2pkh_prevout".to_string(), + ), + p2ms: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2ms_prevout".to_string(), + ), + p2sh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2sh_prevout".to_string(), + ), + p2wpkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2wpkh_prevout".to_string(), + ), + p2wsh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2wsh_prevout".to_string(), + ), + p2tr: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2tr_prevout".to_string(), + ), + p2a: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_p2a_prevout".to_string(), + ), + unknown: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_unknown_outputs_prevout".to_string(), + ), + empty: AverageBlockCumulativeSumPattern::new( + client.clone(), + "tx_count_with_empty_outputs_prevout".to_string(), + ), } } } @@ -4756,7 +6521,10 @@ impl SeriesTree_Outputs { spent: SeriesTree_Outputs_Spent::new(client.clone(), format!("{base_path}_spent")), count: SeriesTree_Outputs_Count::new(client.clone(), format!("{base_path}_count")), per_sec: _1m1w1y24hPattern::new(client.clone(), "outputs_per_sec".to_string()), - unspent: SeriesTree_Outputs_Unspent::new(client.clone(), format!("{base_path}_unspent")), + unspent: SeriesTree_Outputs_Unspent::new( + client.clone(), + format!("{base_path}_unspent"), + ), by_type: SeriesTree_Outputs_ByType::new(client.clone(), format!("{base_path}_by_type")), value: SeriesTree_Outputs_Value::new(client.clone(), format!("{base_path}_value")), } @@ -4775,7 +6543,10 @@ pub struct SeriesTree_Outputs_Raw { impl SeriesTree_Outputs_Raw { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_txout_index: SeriesPattern18::new(client.clone(), "first_txout_index".to_string()), + first_txout_index: SeriesPattern18::new( + client.clone(), + "first_txout_index".to_string(), + ), value: SeriesPattern21::new(client.clone(), "value".to_string()), output_type: SeriesPattern21::new(client.clone(), "output_type".to_string()), type_index: SeriesPattern21::new(client.clone(), "type_index".to_string()), @@ -4835,11 +6606,26 @@ pub struct SeriesTree_Outputs_ByType { impl SeriesTree_Outputs_ByType { pub fn new(client: Arc, base_path: String) -> Self { Self { - output_count: SeriesTree_Outputs_ByType_OutputCount::new(client.clone(), format!("{base_path}_output_count")), - spendable_output_count: AverageBlockCumulativeSumPattern::new(client.clone(), "spendable_output_count".to_string()), - output_share: SeriesTree_Outputs_ByType_OutputShare::new(client.clone(), format!("{base_path}_output_share")), - tx_count: AllEmptyOpP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern::new(client.clone(), "tx_count".to_string()), - tx_share: EmptyOpP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern2::new(client.clone(), "tx_share_with".to_string()), + output_count: SeriesTree_Outputs_ByType_OutputCount::new( + client.clone(), + format!("{base_path}_output_count"), + ), + spendable_output_count: AverageBlockCumulativeSumPattern::new( + client.clone(), + "spendable_output_count".to_string(), + ), + output_share: SeriesTree_Outputs_ByType_OutputShare::new( + client.clone(), + format!("{base_path}_output_share"), + ), + tx_count: AllEmptyOpP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern::new( + client.clone(), + "tx_count".to_string(), + ), + tx_share: EmptyOpP2aP2msP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshUnknownPattern2::new( + client.clone(), + "tx_share_with".to_string(), + ), } } } @@ -4864,19 +6650,58 @@ pub struct SeriesTree_Outputs_ByType_OutputCount { impl SeriesTree_Outputs_ByType_OutputCount { pub fn new(client: Arc, base_path: String) -> Self { Self { - all: AverageBlockCumulativeSumPattern::new(client.clone(), "output_count_bis".to_string()), - p2pk65: AverageBlockCumulativeSumPattern::new(client.clone(), "p2pk65_output_count".to_string()), - p2pk33: AverageBlockCumulativeSumPattern::new(client.clone(), "p2pk33_output_count".to_string()), - p2pkh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2pkh_output_count".to_string()), - p2ms: AverageBlockCumulativeSumPattern::new(client.clone(), "p2ms_output_count".to_string()), - p2sh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2sh_output_count".to_string()), - p2wpkh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2wpkh_output_count".to_string()), - p2wsh: AverageBlockCumulativeSumPattern::new(client.clone(), "p2wsh_output_count".to_string()), - p2tr: AverageBlockCumulativeSumPattern::new(client.clone(), "p2tr_output_count".to_string()), - p2a: AverageBlockCumulativeSumPattern::new(client.clone(), "p2a_output_count".to_string()), - unknown: AverageBlockCumulativeSumPattern::new(client.clone(), "unknown_outputs_output_count".to_string()), - empty: AverageBlockCumulativeSumPattern::new(client.clone(), "empty_outputs_output_count".to_string()), - op_return: AverageBlockCumulativeSumPattern::new(client.clone(), "op_return_output_count".to_string()), + all: AverageBlockCumulativeSumPattern::new( + client.clone(), + "output_count_bis".to_string(), + ), + p2pk65: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2pk65_output_count".to_string(), + ), + p2pk33: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2pk33_output_count".to_string(), + ), + p2pkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2pkh_output_count".to_string(), + ), + p2ms: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2ms_output_count".to_string(), + ), + p2sh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2sh_output_count".to_string(), + ), + p2wpkh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2wpkh_output_count".to_string(), + ), + p2wsh: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2wsh_output_count".to_string(), + ), + p2tr: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2tr_output_count".to_string(), + ), + p2a: AverageBlockCumulativeSumPattern::new( + client.clone(), + "p2a_output_count".to_string(), + ), + unknown: AverageBlockCumulativeSumPattern::new( + client.clone(), + "unknown_outputs_output_count".to_string(), + ), + empty: AverageBlockCumulativeSumPattern::new( + client.clone(), + "empty_outputs_output_count".to_string(), + ), + op_return: AverageBlockCumulativeSumPattern::new( + client.clone(), + "op_return_output_count".to_string(), + ), } } } @@ -4900,18 +6725,54 @@ pub struct SeriesTree_Outputs_ByType_OutputShare { impl SeriesTree_Outputs_ByType_OutputShare { pub fn new(client: Arc, base_path: String) -> Self { Self { - p2pk65: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2pk65_output_share".to_string()), - p2pk33: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2pk33_output_share".to_string()), - p2pkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2pkh_output_share".to_string()), - p2ms: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2ms_output_share".to_string()), - p2sh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2sh_output_share".to_string()), - p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2wpkh_output_share".to_string()), - p2wsh: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2wsh_output_share".to_string()), - p2tr: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2tr_output_share".to_string()), - p2a: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "p2a_output_share".to_string()), - unknown: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "unknown_outputs_output_share".to_string()), - empty: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "empty_outputs_output_share".to_string()), - op_return: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "op_return_output_share".to_string()), + p2pk65: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2pk65_output_share".to_string(), + ), + p2pk33: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2pk33_output_share".to_string(), + ), + p2pkh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2pkh_output_share".to_string(), + ), + p2ms: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2ms_output_share".to_string(), + ), + p2sh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2sh_output_share".to_string(), + ), + p2wpkh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2wpkh_output_share".to_string(), + ), + p2wsh: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2wsh_output_share".to_string(), + ), + p2tr: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2tr_output_share".to_string(), + ), + p2a: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "p2a_output_share".to_string(), + ), + unknown: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "unknown_outputs_output_share".to_string(), + ), + empty: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "empty_outputs_output_share".to_string(), + ), + op_return: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "op_return_output_share".to_string(), + ), } } } @@ -4952,16 +6813,34 @@ impl SeriesTree_Addrs { raw: SeriesTree_Addrs_Raw::new(client.clone(), format!("{base_path}_raw")), indexes: SeriesTree_Addrs_Indexes::new(client.clone(), format!("{base_path}_indexes")), data: SeriesTree_Addrs_Data::new(client.clone(), format!("{base_path}_data")), - funded: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new(client.clone(), "addr_count".to_string()), - empty: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new(client.clone(), "empty_addr_count".to_string()), - activity: SeriesTree_Addrs_Activity::new(client.clone(), format!("{base_path}_activity")), - total: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new(client.clone(), "total_addr_count".to_string()), - new: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new(client.clone(), "new_addr_count".to_string()), + funded: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new( + client.clone(), + "addr_count".to_string(), + ), + empty: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new( + client.clone(), + "empty_addr_count".to_string(), + ), + activity: SeriesTree_Addrs_Activity::new( + client.clone(), + format!("{base_path}_activity"), + ), + total: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4::new( + client.clone(), + "total_addr_count".to_string(), + ), + new: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new( + client.clone(), + "new_addr_count".to_string(), + ), reused: SeriesTree_Addrs_Reused::new(client.clone(), format!("{base_path}_reused")), respent: SeriesTree_Addrs_Respent::new(client.clone(), format!("{base_path}_respent")), exposed: SeriesTree_Addrs_Exposed::new(client.clone(), format!("{base_path}_exposed")), delta: SeriesTree_Addrs_Delta::new(client.clone(), format!("{base_path}_delta")), - avg_amount: SeriesTree_Addrs_AvgAmount::new(client.clone(), format!("{base_path}_avg_amount")), + avg_amount: SeriesTree_Addrs_AvgAmount::new( + client.clone(), + format!("{base_path}_avg_amount"), + ), } } } @@ -5002,7 +6881,10 @@ pub struct SeriesTree_Addrs_Raw_P2pk65 { impl SeriesTree_Addrs_Raw_P2pk65 { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_index: SeriesPattern18::new(client.clone(), "first_p2pk65_addr_index".to_string()), + first_index: SeriesPattern18::new( + client.clone(), + "first_p2pk65_addr_index".to_string(), + ), bytes: SeriesPattern27::new(client.clone(), "p2pk65_bytes".to_string()), } } @@ -5017,7 +6899,10 @@ pub struct SeriesTree_Addrs_Raw_P2pk33 { impl SeriesTree_Addrs_Raw_P2pk33 { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_index: SeriesPattern18::new(client.clone(), "first_p2pk33_addr_index".to_string()), + first_index: SeriesPattern18::new( + client.clone(), + "first_p2pk33_addr_index".to_string(), + ), bytes: SeriesPattern26::new(client.clone(), "p2pk33_bytes".to_string()), } } @@ -5062,7 +6947,10 @@ pub struct SeriesTree_Addrs_Raw_P2wpkh { impl SeriesTree_Addrs_Raw_P2wpkh { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_index: SeriesPattern18::new(client.clone(), "first_p2wpkh_addr_index".to_string()), + first_index: SeriesPattern18::new( + client.clone(), + "first_p2wpkh_addr_index".to_string(), + ), bytes: SeriesPattern31::new(client.clone(), "p2wpkh_bytes".to_string()), } } @@ -5176,14 +7064,38 @@ impl SeriesTree_Addrs_Activity { pub fn new(client: Arc, base_path: String) -> Self { Self { all: SeriesTree_Addrs_Activity_All::new(client.clone(), format!("{base_path}_all")), - p2pk65: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2pk65".to_string()), - p2pk33: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2pk33".to_string()), - p2pkh: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2pkh".to_string()), - p2sh: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2sh".to_string()), - p2wpkh: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2wpkh".to_string()), - p2wsh: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2wsh".to_string()), - p2tr: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2tr".to_string()), - p2a: ActiveBidirectionalReactivatedReceivingSendingPattern::new(client.clone(), "p2a".to_string()), + p2pk65: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2pk65".to_string(), + ), + p2pk33: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2pk33".to_string(), + ), + p2pkh: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2pkh".to_string(), + ), + p2sh: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2sh".to_string(), + ), + p2wpkh: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2wpkh".to_string(), + ), + p2wsh: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2wsh".to_string(), + ), + p2tr: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2tr".to_string(), + ), + p2a: ActiveBidirectionalReactivatedReceivingSendingPattern::new( + client.clone(), + "p2a".to_string(), + ), } } } @@ -5200,10 +7112,16 @@ pub struct SeriesTree_Addrs_Activity_All { impl SeriesTree_Addrs_Activity_All { pub fn new(client: Arc, base_path: String) -> Self { Self { - reactivated: _1m1w1y24hBlockPattern::new(client.clone(), "reactivated_addrs".to_string()), + reactivated: _1m1w1y24hBlockPattern::new( + client.clone(), + "reactivated_addrs".to_string(), + ), sending: _1m1w1y24hBlockPattern::new(client.clone(), "sending_addrs".to_string()), receiving: _1m1w1y24hBlockPattern::new(client.clone(), "receiving_addrs".to_string()), - bidirectional: _1m1w1y24hBlockPattern::new(client.clone(), "bidirectional_addrs".to_string()), + bidirectional: _1m1w1y24hBlockPattern::new( + client.clone(), + "bidirectional_addrs".to_string(), + ), active: _1m1w1y24hBlockPattern::new(client.clone(), "active_addrs".to_string()), } } @@ -5220,8 +7138,14 @@ impl SeriesTree_Addrs_Reused { pub fn new(client: Arc, base_path: String) -> Self { Self { count: FundedTotalPattern::new(client.clone(), "reused_addr_count".to_string()), - events: SeriesTree_Addrs_Reused_Events::new(client.clone(), format!("{base_path}_events")), - supply: SeriesTree_Addrs_Reused_Supply::new(client.clone(), format!("{base_path}_supply")), + events: SeriesTree_Addrs_Reused_Events::new( + client.clone(), + format!("{base_path}_events"), + ), + supply: SeriesTree_Addrs_Reused_Supply::new( + client.clone(), + format!("{base_path}_supply"), + ), } } } @@ -5240,13 +7164,34 @@ pub struct SeriesTree_Addrs_Reused_Events { impl SeriesTree_Addrs_Reused_Events { pub fn new(client: Arc, base_path: String) -> Self { Self { - output_to_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new(client.clone(), "output_to_reused_addr_count".to_string()), - output_to_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new(client.clone(), "output_to_reused_addr_share".to_string()), - spendable_output_to_reused_addr_share: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "spendable_output_to_reused_addr_share".to_string()), - input_from_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new(client.clone(), "input_from_reused_addr_count".to_string()), - input_from_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new(client.clone(), "input_from_reused_addr_share".to_string()), - active_reused_addr_count: _1m1w1y24hBlockPattern::new(client.clone(), "active_reused_addr_count".to_string()), - active_reused_addr_share: _1m1w1y24hBlockPattern2::new(client.clone(), "active_reused_addr_share".to_string()), + output_to_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new( + client.clone(), + "output_to_reused_addr_count".to_string(), + ), + output_to_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new( + client.clone(), + "output_to_reused_addr_share".to_string(), + ), + spendable_output_to_reused_addr_share: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "spendable_output_to_reused_addr_share".to_string(), + ), + input_from_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new( + client.clone(), + "input_from_reused_addr_count".to_string(), + ), + input_from_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new( + client.clone(), + "input_from_reused_addr_share".to_string(), + ), + active_reused_addr_count: _1m1w1y24hBlockPattern::new( + client.clone(), + "active_reused_addr_count".to_string(), + ), + active_reused_addr_share: _1m1w1y24hBlockPattern2::new( + client.clone(), + "active_reused_addr_share".to_string(), + ), } } } @@ -5269,15 +7214,39 @@ impl SeriesTree_Addrs_Reused_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { all: BtcCentsSatsUsdPattern::new(client.clone(), "reused_addr_supply".to_string()), - p2pk65: BtcCentsSatsUsdPattern::new(client.clone(), "p2pk65_reused_addr_supply".to_string()), - p2pk33: BtcCentsSatsUsdPattern::new(client.clone(), "p2pk33_reused_addr_supply".to_string()), - p2pkh: BtcCentsSatsUsdPattern::new(client.clone(), "p2pkh_reused_addr_supply".to_string()), - p2sh: BtcCentsSatsUsdPattern::new(client.clone(), "p2sh_reused_addr_supply".to_string()), - p2wpkh: BtcCentsSatsUsdPattern::new(client.clone(), "p2wpkh_reused_addr_supply".to_string()), - p2wsh: BtcCentsSatsUsdPattern::new(client.clone(), "p2wsh_reused_addr_supply".to_string()), - p2tr: BtcCentsSatsUsdPattern::new(client.clone(), "p2tr_reused_addr_supply".to_string()), + p2pk65: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pk65_reused_addr_supply".to_string(), + ), + p2pk33: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pk33_reused_addr_supply".to_string(), + ), + p2pkh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pkh_reused_addr_supply".to_string(), + ), + p2sh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2sh_reused_addr_supply".to_string(), + ), + p2wpkh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2wpkh_reused_addr_supply".to_string(), + ), + p2wsh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2wsh_reused_addr_supply".to_string(), + ), + p2tr: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2tr_reused_addr_supply".to_string(), + ), p2a: BtcCentsSatsUsdPattern::new(client.clone(), "p2a_reused_addr_supply".to_string()), - share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5::new(client.clone(), "reused_addr_supply_share".to_string()), + share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5::new( + client.clone(), + "reused_addr_supply_share".to_string(), + ), } } } @@ -5293,8 +7262,14 @@ impl SeriesTree_Addrs_Respent { pub fn new(client: Arc, base_path: String) -> Self { Self { count: FundedTotalPattern::new(client.clone(), "respent_addr_count".to_string()), - events: SeriesTree_Addrs_Respent_Events::new(client.clone(), format!("{base_path}_events")), - supply: SeriesTree_Addrs_Respent_Supply::new(client.clone(), format!("{base_path}_supply")), + events: SeriesTree_Addrs_Respent_Events::new( + client.clone(), + format!("{base_path}_events"), + ), + supply: SeriesTree_Addrs_Respent_Supply::new( + client.clone(), + format!("{base_path}_supply"), + ), } } } @@ -5313,13 +7288,34 @@ pub struct SeriesTree_Addrs_Respent_Events { impl SeriesTree_Addrs_Respent_Events { pub fn new(client: Arc, base_path: String) -> Self { Self { - output_to_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new(client.clone(), "output_to_respent_addr_count".to_string()), - output_to_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new(client.clone(), "output_to_respent_addr_share".to_string()), - spendable_output_to_reused_addr_share: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "spendable_output_to_respent_addr_share".to_string()), - input_from_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new(client.clone(), "input_from_respent_addr_count".to_string()), - input_from_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new(client.clone(), "input_from_respent_addr_share".to_string()), - active_reused_addr_count: _1m1w1y24hBlockPattern::new(client.clone(), "active_respent_addr_count".to_string()), - active_reused_addr_share: _1m1w1y24hBlockPattern2::new(client.clone(), "active_respent_addr_share".to_string()), + output_to_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new( + client.clone(), + "output_to_respent_addr_count".to_string(), + ), + output_to_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new( + client.clone(), + "output_to_respent_addr_share".to_string(), + ), + spendable_output_to_reused_addr_share: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "spendable_output_to_respent_addr_share".to_string(), + ), + input_from_reused_addr_count: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern6::new( + client.clone(), + "input_from_respent_addr_count".to_string(), + ), + input_from_reused_addr_share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern7::new( + client.clone(), + "input_from_respent_addr_share".to_string(), + ), + active_reused_addr_count: _1m1w1y24hBlockPattern::new( + client.clone(), + "active_respent_addr_count".to_string(), + ), + active_reused_addr_share: _1m1w1y24hBlockPattern2::new( + client.clone(), + "active_respent_addr_share".to_string(), + ), } } } @@ -5342,15 +7338,39 @@ impl SeriesTree_Addrs_Respent_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { all: BtcCentsSatsUsdPattern::new(client.clone(), "respent_addr_supply".to_string()), - p2pk65: BtcCentsSatsUsdPattern::new(client.clone(), "p2pk65_respent_addr_supply".to_string()), - p2pk33: BtcCentsSatsUsdPattern::new(client.clone(), "p2pk33_respent_addr_supply".to_string()), - p2pkh: BtcCentsSatsUsdPattern::new(client.clone(), "p2pkh_respent_addr_supply".to_string()), - p2sh: BtcCentsSatsUsdPattern::new(client.clone(), "p2sh_respent_addr_supply".to_string()), - p2wpkh: BtcCentsSatsUsdPattern::new(client.clone(), "p2wpkh_respent_addr_supply".to_string()), - p2wsh: BtcCentsSatsUsdPattern::new(client.clone(), "p2wsh_respent_addr_supply".to_string()), - p2tr: BtcCentsSatsUsdPattern::new(client.clone(), "p2tr_respent_addr_supply".to_string()), + p2pk65: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pk65_respent_addr_supply".to_string(), + ), + p2pk33: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pk33_respent_addr_supply".to_string(), + ), + p2pkh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pkh_respent_addr_supply".to_string(), + ), + p2sh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2sh_respent_addr_supply".to_string(), + ), + p2wpkh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2wpkh_respent_addr_supply".to_string(), + ), + p2wsh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2wsh_respent_addr_supply".to_string(), + ), + p2tr: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2tr_respent_addr_supply".to_string(), + ), p2a: BtcCentsSatsUsdPattern::new(client.clone(), "p2a_respent_addr_supply".to_string()), - share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5::new(client.clone(), "respent_addr_supply_share".to_string()), + share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5::new( + client.clone(), + "respent_addr_supply_share".to_string(), + ), } } } @@ -5365,7 +7385,10 @@ impl SeriesTree_Addrs_Exposed { pub fn new(client: Arc, base_path: String) -> Self { Self { count: FundedTotalPattern::new(client.clone(), "exposed_addr_count".to_string()), - supply: SeriesTree_Addrs_Exposed_Supply::new(client.clone(), format!("{base_path}_supply")), + supply: SeriesTree_Addrs_Exposed_Supply::new( + client.clone(), + format!("{base_path}_supply"), + ), } } } @@ -5388,15 +7411,39 @@ impl SeriesTree_Addrs_Exposed_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { all: BtcCentsSatsUsdPattern::new(client.clone(), "exposed_addr_supply".to_string()), - p2pk65: BtcCentsSatsUsdPattern::new(client.clone(), "p2pk65_exposed_addr_supply".to_string()), - p2pk33: BtcCentsSatsUsdPattern::new(client.clone(), "p2pk33_exposed_addr_supply".to_string()), - p2pkh: BtcCentsSatsUsdPattern::new(client.clone(), "p2pkh_exposed_addr_supply".to_string()), - p2sh: BtcCentsSatsUsdPattern::new(client.clone(), "p2sh_exposed_addr_supply".to_string()), - p2wpkh: BtcCentsSatsUsdPattern::new(client.clone(), "p2wpkh_exposed_addr_supply".to_string()), - p2wsh: BtcCentsSatsUsdPattern::new(client.clone(), "p2wsh_exposed_addr_supply".to_string()), - p2tr: BtcCentsSatsUsdPattern::new(client.clone(), "p2tr_exposed_addr_supply".to_string()), + p2pk65: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pk65_exposed_addr_supply".to_string(), + ), + p2pk33: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pk33_exposed_addr_supply".to_string(), + ), + p2pkh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2pkh_exposed_addr_supply".to_string(), + ), + p2sh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2sh_exposed_addr_supply".to_string(), + ), + p2wpkh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2wpkh_exposed_addr_supply".to_string(), + ), + p2wsh: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2wsh_exposed_addr_supply".to_string(), + ), + p2tr: BtcCentsSatsUsdPattern::new( + client.clone(), + "p2tr_exposed_addr_supply".to_string(), + ), p2a: BtcCentsSatsUsdPattern::new(client.clone(), "p2a_exposed_addr_supply".to_string()), - share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5::new(client.clone(), "exposed_addr_supply_share".to_string()), + share: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern5::new( + client.clone(), + "exposed_addr_supply_share".to_string(), + ), } } } @@ -5484,7 +7531,10 @@ impl SeriesTree_Scripts_Raw { Self { empty: SeriesTree_Scripts_Raw_Empty::new(client.clone(), format!("{base_path}_empty")), p2ms: SeriesTree_Scripts_Raw_P2ms::new(client.clone(), format!("{base_path}_p2ms")), - unknown: SeriesTree_Scripts_Raw_Unknown::new(client.clone(), format!("{base_path}_unknown")), + unknown: SeriesTree_Scripts_Raw_Unknown::new( + client.clone(), + format!("{base_path}_unknown"), + ), } } } @@ -5498,7 +7548,10 @@ pub struct SeriesTree_Scripts_Raw_Empty { impl SeriesTree_Scripts_Raw_Empty { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_index: SeriesPattern18::new(client.clone(), "first_empty_output_index".to_string()), + first_index: SeriesPattern18::new( + client.clone(), + "first_empty_output_index".to_string(), + ), to_tx_index: SeriesPattern22::new(client.clone(), "tx_index".to_string()), } } @@ -5514,7 +7567,10 @@ pub struct SeriesTree_Scripts_Raw_P2ms { impl SeriesTree_Scripts_Raw_P2ms { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_index: SeriesPattern18::new(client.clone(), "first_p2ms_output_index".to_string()), + first_index: SeriesPattern18::new( + client.clone(), + "first_p2ms_output_index".to_string(), + ), to_tx_index: SeriesPattern25::new(client.clone(), "tx_index".to_string()), legacy_sigops: SeriesPattern25::new(client.clone(), "p2ms_legacy_sigops".to_string()), } @@ -5531,9 +7587,15 @@ pub struct SeriesTree_Scripts_Raw_Unknown { impl SeriesTree_Scripts_Raw_Unknown { pub fn new(client: Arc, base_path: String) -> Self { Self { - first_index: SeriesPattern18::new(client.clone(), "first_unknown_output_index".to_string()), + first_index: SeriesPattern18::new( + client.clone(), + "first_unknown_output_index".to_string(), + ), to_tx_index: SeriesPattern33::new(client.clone(), "tx_index".to_string()), - legacy_sigops: SeriesPattern33::new(client.clone(), "unknown_legacy_sigops".to_string()), + legacy_sigops: SeriesPattern33::new( + client.clone(), + "unknown_legacy_sigops".to_string(), + ), } } } @@ -5551,7 +7613,10 @@ impl SeriesTree_OpReturn { Self { raw: SeriesTree_OpReturn_Raw::new(client.clone(), format!("{base_path}_raw")), total: SeriesTree_OpReturn_Total::new(client.clone(), format!("{base_path}_total")), - by_kind: SeriesTree_OpReturn_ByKind::new(client.clone(), format!("{base_path}_by_kind")), + by_kind: SeriesTree_OpReturn_ByKind::new( + client.clone(), + format!("{base_path}_by_kind"), + ), policy: SeriesTree_OpReturn_Policy::new(client.clone(), format!("{base_path}_policy")), } } @@ -5571,7 +7636,10 @@ impl SeriesTree_OpReturn_Raw { first_index: SeriesPattern18::new(client.clone(), "first_op_return_index".to_string()), to_tx_index: SeriesPattern23::new(client.clone(), "tx_index".to_string()), kind: SeriesPattern23::new(client.clone(), "kind".to_string()), - post_op_return_bytes: SeriesPattern23::new(client.clone(), "op_return_post_op_return_bytes".to_string()), + post_op_return_bytes: SeriesPattern23::new( + client.clone(), + "op_return_post_op_return_bytes".to_string(), + ), } } } @@ -5589,12 +7657,30 @@ pub struct SeriesTree_OpReturn_Total { impl SeriesTree_OpReturn_Total { pub fn new(client: Arc, base_path: String) -> Self { Self { - data_bytes: AverageBlockCumulativeSumPattern::new(client.clone(), "op_return_data_bytes".to_string()), - tx_count: AverageBlockCumulativeSumPattern::new(client.clone(), "op_return_tx_count".to_string()), - tx_vsize: AverageBlockCumulativeSumPattern::new(client.clone(), "op_return_tx_vsize".to_string()), - fees: AverageBlockCumulativeSumPattern::new(client.clone(), "op_return_fees".to_string()), - chain_share: PercentPpmRatioPattern2::new(client.clone(), "op_return_chain_share".to_string()), - fee_share: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "op_return_fee_share".to_string()), + data_bytes: AverageBlockCumulativeSumPattern::new( + client.clone(), + "op_return_data_bytes".to_string(), + ), + tx_count: AverageBlockCumulativeSumPattern::new( + client.clone(), + "op_return_tx_count".to_string(), + ), + tx_vsize: AverageBlockCumulativeSumPattern::new( + client.clone(), + "op_return_tx_vsize".to_string(), + ), + fees: AverageBlockCumulativeSumPattern::new( + client.clone(), + "op_return_fees".to_string(), + ), + chain_share: PercentPpmRatioPattern2::new( + client.clone(), + "op_return_chain_share".to_string(), + ), + fee_share: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "op_return_fee_share".to_string(), + ), } } } @@ -5629,29 +7715,98 @@ pub struct SeriesTree_OpReturn_ByKind { impl SeriesTree_OpReturn_ByKind { pub fn new(client: Arc, base_path: String) -> Self { Self { - runes: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_runes".to_string()), - veri_block: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_veri_block".to_string()), - omni: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_omni".to_string()), - stacks: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_stacks".to_string()), - blockstack: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_blockstack".to_string()), - colu: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_colu".to_string()), - open_assets: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_open_assets".to_string()), - komodo: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_komodo".to_string()), - coin_spark: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_coin_spark".to_string()), - poet: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_poet".to_string()), - docproof: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_docproof".to_string()), - open_timestamps: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_open_timestamps".to_string()), - factom: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_factom".to_string()), - eternity_wall: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_eternity_wall".to_string()), - memo: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_memo".to_string()), - bitproof: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_bitproof".to_string()), - ascribe: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_ascribe".to_string()), - stampery: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_stampery".to_string()), - epobc: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_epobc".to_string()), - bare_hash: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_bare_hash".to_string()), - text: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_text".to_string()), - empty: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_empty".to_string()), - unknown: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_unknown".to_string()), + runes: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_runes".to_string(), + ), + veri_block: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_veri_block".to_string(), + ), + omni: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_omni".to_string(), + ), + stacks: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_stacks".to_string(), + ), + blockstack: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_blockstack".to_string(), + ), + colu: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_colu".to_string(), + ), + open_assets: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_open_assets".to_string(), + ), + komodo: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_komodo".to_string(), + ), + coin_spark: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_coin_spark".to_string(), + ), + poet: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_poet".to_string(), + ), + docproof: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_docproof".to_string(), + ), + open_timestamps: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_open_timestamps".to_string(), + ), + factom: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_factom".to_string(), + ), + eternity_wall: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_eternity_wall".to_string(), + ), + memo: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_memo".to_string(), + ), + bitproof: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_bitproof".to_string(), + ), + ascribe: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_ascribe".to_string(), + ), + stampery: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_stampery".to_string(), + ), + epobc: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_epobc".to_string(), + ), + bare_hash: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_bare_hash".to_string(), + ), + text: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_text".to_string(), + ), + empty: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_empty".to_string(), + ), + unknown: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_unknown".to_string(), + ), } } } @@ -5667,10 +7822,22 @@ pub struct SeriesTree_OpReturn_Policy { impl SeriesTree_OpReturn_Policy { pub fn new(client: Arc, base_path: String) -> Self { Self { - pre_v30_standard: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_policy_pre_v30_standard".to_string()), - pre_v30_nonstandard: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_policy_pre_v30_nonstandard".to_string()), - oversized: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_policy_oversized".to_string()), - multiple: ChainDataFeeFeesOutputTxPattern::new(client.clone(), "op_return_policy_multiple".to_string()), + pre_v30_standard: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_policy_pre_v30_standard".to_string(), + ), + pre_v30_nonstandard: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_policy_pre_v30_nonstandard".to_string(), + ), + oversized: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_policy_oversized".to_string(), + ), + multiple: ChainDataFeeFeesOutputTxPattern::new( + client.clone(), + "op_return_policy_multiple".to_string(), + ), } } } @@ -5685,7 +7852,10 @@ impl SeriesTree_Mining { pub fn new(client: Arc, base_path: String) -> Self { Self { rewards: SeriesTree_Mining_Rewards::new(client.clone(), format!("{base_path}_rewards")), - hashrate: SeriesTree_Mining_Hashrate::new(client.clone(), format!("{base_path}_hashrate")), + hashrate: SeriesTree_Mining_Hashrate::new( + client.clone(), + format!("{base_path}_hashrate"), + ), } } } @@ -5702,8 +7872,14 @@ pub struct SeriesTree_Mining_Rewards { impl SeriesTree_Mining_Rewards { pub fn new(client: Arc, base_path: String) -> Self { Self { - coinbase: AverageBlockCumulativeSumPattern2::new(client.clone(), "coinbase".to_string()), - subsidy: SeriesTree_Mining_Rewards_Subsidy::new(client.clone(), format!("{base_path}_subsidy")), + coinbase: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "coinbase".to_string(), + ), + subsidy: SeriesTree_Mining_Rewards_Subsidy::new( + client.clone(), + format!("{base_path}_subsidy"), + ), fees: SeriesTree_Mining_Rewards_Fees::new(client.clone(), format!("{base_path}_fees")), output_volume: SeriesPattern18::new(client.clone(), "output_volume".to_string()), unclaimed: BlockCumulativePattern::new(client.clone(), "unclaimed_rewards".to_string()), @@ -5724,10 +7900,16 @@ impl SeriesTree_Mining_Rewards_Subsidy { pub fn new(client: Arc, base_path: String) -> Self { Self { block: BtcCentsSatsUsdPattern3::new(client.clone(), "subsidy".to_string()), - cumulative: BtcCentsSatsUsdPattern::new(client.clone(), "subsidy_cumulative".to_string()), + cumulative: BtcCentsSatsUsdPattern::new( + client.clone(), + "subsidy_cumulative".to_string(), + ), sum: _1m1w1y24hPattern4::new(client.clone(), "subsidy_sum".to_string()), average: _1m1w1y24hPattern3::new(client.clone(), "subsidy_average".to_string()), - dominance: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "subsidy_dominance".to_string()), + dominance: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "subsidy_dominance".to_string(), + ), } } } @@ -5763,8 +7945,14 @@ impl SeriesTree_Mining_Rewards_Fees { median: _1m1w1y24hPattern4::new(client.clone(), "fees_median".to_string()), pct75: _1m1w1y24hPattern4::new(client.clone(), "fees_pct75".to_string()), pct90: _1m1w1y24hPattern4::new(client.clone(), "fees_pct90".to_string()), - dominance: _1m1w1y24hPercentPpmRatioPattern::new(client.clone(), "fee_dominance".to_string()), - to_subsidy: SeriesTree_Mining_Rewards_Fees_ToSubsidy::new(client.clone(), format!("{base_path}_to_subsidy")), + dominance: _1m1w1y24hPercentPpmRatioPattern::new( + client.clone(), + "fee_dominance".to_string(), + ), + to_subsidy: SeriesTree_Mining_Rewards_Fees_ToSubsidy::new( + client.clone(), + format!("{base_path}_to_subsidy"), + ), } } } @@ -5817,9 +8005,15 @@ impl SeriesTree_Mining_Hashrate_Rate { pub fn new(client: Arc, base_path: String) -> Self { Self { base: SeriesPattern1::new(client.clone(), "hash_rate".to_string()), - sma: SeriesTree_Mining_Hashrate_Rate_Sma::new(client.clone(), format!("{base_path}_sma")), + sma: SeriesTree_Mining_Hashrate_Rate_Sma::new( + client.clone(), + format!("{base_path}_sma"), + ), ath: SeriesPattern1::new(client.clone(), "hash_rate_ath".to_string()), - drawdown: PercentPpmRatioPattern3::new(client.clone(), "hash_rate_drawdown".to_string()), + drawdown: PercentPpmRatioPattern3::new( + client.clone(), + "hash_rate_drawdown".to_string(), + ), } } } @@ -5858,14 +8052,26 @@ pub struct SeriesTree_Cointime { impl SeriesTree_Cointime { pub fn new(client: Arc, base_path: String) -> Self { Self { - activity: SeriesTree_Cointime_Activity::new(client.clone(), format!("{base_path}_activity")), - age_range: SeriesTree_Cointime_AgeRange::new(client.clone(), format!("{base_path}_age_range")), + activity: SeriesTree_Cointime_Activity::new( + client.clone(), + format!("{base_path}_activity"), + ), + age_range: SeriesTree_Cointime_AgeRange::new( + client.clone(), + format!("{base_path}_age_range"), + ), supply: SeriesTree_Cointime_Supply::new(client.clone(), format!("{base_path}_supply")), value: SeriesTree_Cointime_Value::new(client.clone(), format!("{base_path}_value")), cap: SeriesTree_Cointime_Cap::new(client.clone(), format!("{base_path}_cap")), prices: SeriesTree_Cointime_Prices::new(client.clone(), format!("{base_path}_prices")), - adjusted: SeriesTree_Cointime_Adjusted::new(client.clone(), format!("{base_path}_adjusted")), - reserve_risk: SeriesTree_Cointime_ReserveRisk::new(client.clone(), format!("{base_path}_reserve_risk")), + adjusted: SeriesTree_Cointime_Adjusted::new( + client.clone(), + format!("{base_path}_adjusted"), + ), + reserve_risk: SeriesTree_Cointime_ReserveRisk::new( + client.clone(), + format!("{base_path}_reserve_risk"), + ), } } } @@ -5883,12 +8089,21 @@ pub struct SeriesTree_Cointime_Activity { impl SeriesTree_Cointime_Activity { pub fn new(client: Arc, base_path: String) -> Self { Self { - coinblocks_created: AverageBlockCumulativeSumPattern::new(client.clone(), "coinblocks_created".to_string()), - coinblocks_stored: AverageBlockCumulativeSumPattern::new(client.clone(), "coinblocks_stored".to_string()), + coinblocks_created: AverageBlockCumulativeSumPattern::new( + client.clone(), + "coinblocks_created".to_string(), + ), + coinblocks_stored: AverageBlockCumulativeSumPattern::new( + client.clone(), + "coinblocks_stored".to_string(), + ), liveliness: SeriesPattern1::new(client.clone(), "liveliness".to_string()), vaultedness: SeriesPattern1::new(client.clone(), "vaultedness".to_string()), ratio: SeriesPattern1::new(client.clone(), "activity_to_vaultedness".to_string()), - coinblocks_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), "coinblocks_destroyed".to_string()), + coinblocks_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + "coinblocks_destroyed".to_string(), + ), } } } @@ -5923,29 +8138,98 @@ pub struct SeriesTree_Cointime_AgeRange { impl SeriesTree_Cointime_AgeRange { pub fn new(client: Arc, base_path: String) -> Self { Self { - under_1h: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_under_1h_old".to_string()), - _1h_to_1d: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1h_to_1d_old".to_string()), - _1d_to_1w: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1d_to_1w_old".to_string()), - _1w_to_1m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1w_to_1m_old".to_string()), - _1m_to_2m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1m_to_2m_old".to_string()), - _2m_to_3m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_2m_to_3m_old".to_string()), - _3m_to_4m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_3m_to_4m_old".to_string()), - _4m_to_5m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_4m_to_5m_old".to_string()), - _5m_to_6m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_5m_to_6m_old".to_string()), - _6m_to_9m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_6m_to_9m_old".to_string()), - _9m_to_1y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_9m_to_1y_old".to_string()), - _1y_to_18m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1y_to_18m_old".to_string()), - _18m_to_2y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_18m_to_2y_old".to_string()), - _2y_to_3y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_2y_to_3y_old".to_string()), - _3y_to_4y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_3y_to_4y_old".to_string()), - _4y_to_5y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_4y_to_5y_old".to_string()), - _5y_to_6y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_5y_to_6y_old".to_string()), - _6y_to_7y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_6y_to_7y_old".to_string()), - _7y_to_8y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_7y_to_8y_old".to_string()), - _8y_to_10y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_8y_to_10y_old".to_string()), - _10y_to_12y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_10y_to_12y_old".to_string()), - _12y_to_15y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_12y_to_15y_old".to_string()), - over_15y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_over_15y_old".to_string()), + under_1h: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_under_1h_old".to_string(), + ), + _1h_to_1d: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_1h_to_1d_old".to_string(), + ), + _1d_to_1w: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_1d_to_1w_old".to_string(), + ), + _1w_to_1m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_1w_to_1m_old".to_string(), + ), + _1m_to_2m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_1m_to_2m_old".to_string(), + ), + _2m_to_3m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_2m_to_3m_old".to_string(), + ), + _3m_to_4m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_3m_to_4m_old".to_string(), + ), + _4m_to_5m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_4m_to_5m_old".to_string(), + ), + _5m_to_6m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_5m_to_6m_old".to_string(), + ), + _6m_to_9m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_6m_to_9m_old".to_string(), + ), + _9m_to_1y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_9m_to_1y_old".to_string(), + ), + _1y_to_18m: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_1y_to_18m_old".to_string(), + ), + _18m_to_2y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_18m_to_2y_old".to_string(), + ), + _2y_to_3y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_2y_to_3y_old".to_string(), + ), + _3y_to_4y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_3y_to_4y_old".to_string(), + ), + _4y_to_5y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_4y_to_5y_old".to_string(), + ), + _5y_to_6y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_5y_to_6y_old".to_string(), + ), + _6y_to_7y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_6y_to_7y_old".to_string(), + ), + _7y_to_8y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_7y_to_8y_old".to_string(), + ), + _8y_to_10y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_8y_to_10y_old".to_string(), + ), + _10y_to_12y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_10y_to_12y_old".to_string(), + ), + _12y_to_15y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_12y_to_15y_old".to_string(), + ), + over_15y: CoindaysLivelinessRatioSupplyVaultednessPattern::new( + client.clone(), + "utxos_over_15y_old".to_string(), + ), } } } @@ -5960,7 +8244,10 @@ impl SeriesTree_Cointime_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { vaulted: BtcCentsSatsUsdPattern::new(client.clone(), "vaulted_supply".to_string()), - active: SeriesTree_Cointime_Supply_Active::new(client.clone(), format!("{base_path}_active")), + active: SeriesTree_Cointime_Supply_Active::new( + client.clone(), + format!("{base_path}_active"), + ), } } } @@ -5981,7 +8268,10 @@ impl SeriesTree_Cointime_Supply_Active { sats: SeriesPattern1::new(client.clone(), "active_supply_sats".to_string()), usd: SeriesPattern1::new(client.clone(), "active_supply_usd".to_string()), cents: SeriesPattern1::new(client.clone(), "active_supply_cents".to_string()), - in_loss: SharePattern2::new(client.clone(), "cointime_supply_in_loss_share".to_string()), + in_loss: SharePattern2::new( + client.clone(), + "cointime_supply_in_loss_share".to_string(), + ), } } } @@ -5997,9 +8287,18 @@ pub struct SeriesTree_Cointime_Value { impl SeriesTree_Cointime_Value { pub fn new(client: Arc, base_path: String) -> Self { Self { - destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), "cointime_value_destroyed".to_string()), - created: AverageBlockCumulativeSumPattern::new(client.clone(), "cointime_value_created".to_string()), - stored: AverageBlockCumulativeSumPattern::new(client.clone(), "cointime_value_stored".to_string()), + destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + "cointime_value_destroyed".to_string(), + ), + created: AverageBlockCumulativeSumPattern::new( + client.clone(), + "cointime_value_created".to_string(), + ), + stored: AverageBlockCumulativeSumPattern::new( + client.clone(), + "cointime_value_stored".to_string(), + ), vocdd: AverageBlockCumulativeSumPattern::new(client.clone(), "vocdd".to_string()), } } @@ -6041,8 +8340,14 @@ impl SeriesTree_Cointime_Prices { Self { vaulted: CentsPpmRatioSatsUsdPattern::new(client.clone(), "vaulted_price".to_string()), active: CentsPpmRatioSatsUsdPattern::new(client.clone(), "active_price".to_string()), - true_market_mean: CentsPpmRatioSatsUsdPattern::new(client.clone(), "true_market_mean".to_string()), - cointime: CentsPpmRatioSatsUsdPattern::new(client.clone(), "cointime_price".to_string()), + true_market_mean: CentsPpmRatioSatsUsdPattern::new( + client.clone(), + "true_market_mean".to_string(), + ), + cointime: CentsPpmRatioSatsUsdPattern::new( + client.clone(), + "cointime_price".to_string(), + ), } } } @@ -6057,9 +8362,18 @@ pub struct SeriesTree_Cointime_Adjusted { impl SeriesTree_Cointime_Adjusted { pub fn new(client: Arc, base_path: String) -> Self { Self { - inflation_rate: PercentPpmRatioPattern::new(client.clone(), "cointime_adj_inflation_rate".to_string()), - tx_velocity_native: SeriesPattern1::new(client.clone(), "cointime_adj_tx_velocity_btc".to_string()), - tx_velocity_fiat: SeriesPattern1::new(client.clone(), "cointime_adj_tx_velocity_usd".to_string()), + inflation_rate: PercentPpmRatioPattern::new( + client.clone(), + "cointime_adj_inflation_rate".to_string(), + ), + tx_velocity_native: SeriesPattern1::new( + client.clone(), + "cointime_adj_tx_velocity_btc".to_string(), + ), + tx_velocity_fiat: SeriesPattern1::new( + client.clone(), + "cointime_adj_tx_velocity_usd".to_string(), + ), } } } @@ -6093,9 +8407,15 @@ pub struct SeriesTree_Coinflow { impl SeriesTree_Coinflow { pub fn new(client: Arc, base_path: String) -> Self { Self { - age_range: SeriesTree_Coinflow_AgeRange::new(client.clone(), format!("{base_path}_age_range")), + age_range: SeriesTree_Coinflow_AgeRange::new( + client.clone(), + format!("{base_path}_age_range"), + ), supply: SeriesTree_Coinflow_Supply::new(client.clone(), format!("{base_path}_supply")), - horizon: SeriesTree_Coinflow_Horizon::new(client.clone(), format!("{base_path}_horizon")), + horizon: SeriesTree_Coinflow_Horizon::new( + client.clone(), + format!("{base_path}_horizon"), + ), cap: CentsUsdPattern3::new(client.clone(), "coinflow_cap".to_string()), price: CentsPpmRatioSatsUsdPattern::new(client.clone(), "coinflow_price".to_string()), } @@ -6132,29 +8452,98 @@ pub struct SeriesTree_Coinflow_AgeRange { impl SeriesTree_Coinflow_AgeRange { pub fn new(client: Arc, base_path: String) -> Self { Self { - under_1h: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_under_1h_old".to_string()), - _1h_to_1d: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1h_to_1d_old".to_string()), - _1d_to_1w: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1d_to_1w_old".to_string()), - _1w_to_1m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1w_to_1m_old".to_string()), - _1m_to_2m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1m_to_2m_old".to_string()), - _2m_to_3m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_2m_to_3m_old".to_string()), - _3m_to_4m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_3m_to_4m_old".to_string()), - _4m_to_5m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_4m_to_5m_old".to_string()), - _5m_to_6m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_5m_to_6m_old".to_string()), - _6m_to_9m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_6m_to_9m_old".to_string()), - _9m_to_1y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_9m_to_1y_old".to_string()), - _1y_to_18m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1y_to_18m_old".to_string()), - _18m_to_2y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_18m_to_2y_old".to_string()), - _2y_to_3y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_2y_to_3y_old".to_string()), - _3y_to_4y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_3y_to_4y_old".to_string()), - _4y_to_5y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_4y_to_5y_old".to_string()), - _5y_to_6y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_5y_to_6y_old".to_string()), - _6y_to_7y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_6y_to_7y_old".to_string()), - _7y_to_8y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_7y_to_8y_old".to_string()), - _8y_to_10y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_8y_to_10y_old".to_string()), - _10y_to_12y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_10y_to_12y_old".to_string()), - _12y_to_15y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_12y_to_15y_old".to_string()), - over_15y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_over_15y_old".to_string()), + under_1h: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_under_1h_old".to_string(), + ), + _1h_to_1d: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_1h_to_1d_old".to_string(), + ), + _1d_to_1w: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_1d_to_1w_old".to_string(), + ), + _1w_to_1m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_1w_to_1m_old".to_string(), + ), + _1m_to_2m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_1m_to_2m_old".to_string(), + ), + _2m_to_3m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_2m_to_3m_old".to_string(), + ), + _3m_to_4m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_3m_to_4m_old".to_string(), + ), + _4m_to_5m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_4m_to_5m_old".to_string(), + ), + _5m_to_6m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_5m_to_6m_old".to_string(), + ), + _6m_to_9m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_6m_to_9m_old".to_string(), + ), + _9m_to_1y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_9m_to_1y_old".to_string(), + ), + _1y_to_18m: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_1y_to_18m_old".to_string(), + ), + _18m_to_2y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_18m_to_2y_old".to_string(), + ), + _2y_to_3y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_2y_to_3y_old".to_string(), + ), + _3y_to_4y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_3y_to_4y_old".to_string(), + ), + _4y_to_5y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_4y_to_5y_old".to_string(), + ), + _5y_to_6y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_5y_to_6y_old".to_string(), + ), + _6y_to_7y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_6y_to_7y_old".to_string(), + ), + _7y_to_8y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_7y_to_8y_old".to_string(), + ), + _8y_to_10y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_8y_to_10y_old".to_string(), + ), + _10y_to_12y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_10y_to_12y_old".to_string(), + ), + _12y_to_15y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_12y_to_15y_old".to_string(), + ), + over_15y: MobilitySpendingSupplyPattern::new( + client.clone(), + "utxos_over_15y_old".to_string(), + ), } } } @@ -6168,7 +8557,10 @@ pub struct SeriesTree_Coinflow_Supply { impl SeriesTree_Coinflow_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { - mobile: SeriesTree_Coinflow_Supply_Mobile::new(client.clone(), format!("{base_path}_mobile")), + mobile: SeriesTree_Coinflow_Supply_Mobile::new( + client.clone(), + format!("{base_path}_mobile"), + ), immobile: BtcCentsSatsUsdPattern::new(client.clone(), "immobile_supply".to_string()), } } @@ -6190,7 +8582,10 @@ impl SeriesTree_Coinflow_Supply_Mobile { sats: SeriesPattern1::new(client.clone(), "mobile_supply_sats".to_string()), usd: SeriesPattern1::new(client.clone(), "mobile_supply_usd".to_string()), cents: SeriesPattern1::new(client.clone(), "mobile_supply_cents".to_string()), - in_loss: SharePattern2::new(client.clone(), "coinflow_supply_in_loss_share".to_string()), + in_loss: SharePattern2::new( + client.clone(), + "coinflow_supply_in_loss_share".to_string(), + ), } } } @@ -6209,13 +8604,34 @@ pub struct SeriesTree_Coinflow_Horizon { impl SeriesTree_Coinflow_Horizon { pub fn new(client: Arc, base_path: String) -> Self { Self { - _8y: SupplyPattern::new(client.clone(), "coinflow_8y_supply_in_loss_share".to_string()), - _4y: SupplyPattern::new(client.clone(), "coinflow_4y_supply_in_loss_share".to_string()), - _2y: SupplyPattern::new(client.clone(), "coinflow_2y_supply_in_loss_share".to_string()), - _1y: SupplyPattern::new(client.clone(), "coinflow_1y_supply_in_loss_share".to_string()), - _6m: SupplyPattern::new(client.clone(), "coinflow_6m_supply_in_loss_share".to_string()), - _3m: SupplyPattern::new(client.clone(), "coinflow_3m_supply_in_loss_share".to_string()), - _1m: SupplyPattern::new(client.clone(), "coinflow_1m_supply_in_loss_share".to_string()), + _8y: SupplyPattern::new( + client.clone(), + "coinflow_8y_supply_in_loss_share".to_string(), + ), + _4y: SupplyPattern::new( + client.clone(), + "coinflow_4y_supply_in_loss_share".to_string(), + ), + _2y: SupplyPattern::new( + client.clone(), + "coinflow_2y_supply_in_loss_share".to_string(), + ), + _1y: SupplyPattern::new( + client.clone(), + "coinflow_1y_supply_in_loss_share".to_string(), + ), + _6m: SupplyPattern::new( + client.clone(), + "coinflow_6m_supply_in_loss_share".to_string(), + ), + _3m: SupplyPattern::new( + client.clone(), + "coinflow_3m_supply_in_loss_share".to_string(), + ), + _1m: SupplyPattern::new( + client.clone(), + "coinflow_1m_supply_in_loss_share".to_string(), + ), } } } @@ -6240,13 +8656,34 @@ impl SeriesTree_Bedrock { raw: FloorLevelLossPattern::new(client.clone(), "bedrock_raw".to_string()), cointime: FloorLevelLossPattern::new(client.clone(), "bedrock_cointime".to_string()), coinflow: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow".to_string()), - coinflow_8y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_8y".to_string()), - coinflow_4y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_4y".to_string()), - coinflow_2y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_2y".to_string()), - coinflow_1y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_1y".to_string()), - coinflow_6m: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_6m".to_string()), - coinflow_3m: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_3m".to_string()), - coinflow_1m: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_1m".to_string()), + coinflow_8y: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_8y".to_string(), + ), + coinflow_4y: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_4y".to_string(), + ), + coinflow_2y: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_2y".to_string(), + ), + coinflow_1y: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_1y".to_string(), + ), + coinflow_6m: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_6m".to_string(), + ), + coinflow_3m: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_3m".to_string(), + ), + coinflow_1m: FloorLevelLossPattern::new( + client.clone(), + "bedrock_coinflow_1m".to_string(), + ), } } } @@ -6329,9 +8766,18 @@ impl SeriesTree_Indexes { addr: SeriesTree_Indexes_Addr::new(client.clone(), format!("{base_path}_addr")), height: SeriesTree_Indexes_Height::new(client.clone(), format!("{base_path}_height")), epoch: SeriesTree_Indexes_Epoch::new(client.clone(), format!("{base_path}_epoch")), - halving: SeriesTree_Indexes_Halving::new(client.clone(), format!("{base_path}_halving")), - minute10: SeriesTree_Indexes_Minute10::new(client.clone(), format!("{base_path}_minute10")), - minute30: SeriesTree_Indexes_Minute30::new(client.clone(), format!("{base_path}_minute30")), + halving: SeriesTree_Indexes_Halving::new( + client.clone(), + format!("{base_path}_halving"), + ), + minute10: SeriesTree_Indexes_Minute10::new( + client.clone(), + format!("{base_path}_minute10"), + ), + minute30: SeriesTree_Indexes_Minute30::new( + client.clone(), + format!("{base_path}_minute30"), + ), hour1: SeriesTree_Indexes_Hour1::new(client.clone(), format!("{base_path}_hour1")), hour4: SeriesTree_Indexes_Hour4::new(client.clone(), format!("{base_path}_hour4")), hour12: SeriesTree_Indexes_Hour12::new(client.clone(), format!("{base_path}_hour12")), @@ -6343,10 +8789,22 @@ impl SeriesTree_Indexes { month6: SeriesTree_Indexes_Month6::new(client.clone(), format!("{base_path}_month6")), year1: SeriesTree_Indexes_Year1::new(client.clone(), format!("{base_path}_year1")), year10: SeriesTree_Indexes_Year10::new(client.clone(), format!("{base_path}_year10")), - tx_index: SeriesTree_Indexes_TxIndex::new(client.clone(), format!("{base_path}_tx_index")), - txin_index: SeriesTree_Indexes_TxinIndex::new(client.clone(), format!("{base_path}_txin_index")), - txout_index: SeriesTree_Indexes_TxoutIndex::new(client.clone(), format!("{base_path}_txout_index")), - timestamp: SeriesTree_Indexes_Timestamp::new(client.clone(), format!("{base_path}_timestamp")), + tx_index: SeriesTree_Indexes_TxIndex::new( + client.clone(), + format!("{base_path}_tx_index"), + ), + txin_index: SeriesTree_Indexes_TxinIndex::new( + client.clone(), + format!("{base_path}_txin_index"), + ), + txout_index: SeriesTree_Indexes_TxoutIndex::new( + client.clone(), + format!("{base_path}_txout_index"), + ), + timestamp: SeriesTree_Indexes_Timestamp::new( + client.clone(), + format!("{base_path}_timestamp"), + ), } } } @@ -6370,18 +8828,33 @@ pub struct SeriesTree_Indexes_Addr { impl SeriesTree_Indexes_Addr { pub fn new(client: Arc, base_path: String) -> Self { Self { - p2pk33: SeriesTree_Indexes_Addr_P2pk33::new(client.clone(), format!("{base_path}_p2pk33")), - p2pk65: SeriesTree_Indexes_Addr_P2pk65::new(client.clone(), format!("{base_path}_p2pk65")), + p2pk33: SeriesTree_Indexes_Addr_P2pk33::new( + client.clone(), + format!("{base_path}_p2pk33"), + ), + p2pk65: SeriesTree_Indexes_Addr_P2pk65::new( + client.clone(), + format!("{base_path}_p2pk65"), + ), p2pkh: SeriesTree_Indexes_Addr_P2pkh::new(client.clone(), format!("{base_path}_p2pkh")), p2sh: SeriesTree_Indexes_Addr_P2sh::new(client.clone(), format!("{base_path}_p2sh")), p2tr: SeriesTree_Indexes_Addr_P2tr::new(client.clone(), format!("{base_path}_p2tr")), - p2wpkh: SeriesTree_Indexes_Addr_P2wpkh::new(client.clone(), format!("{base_path}_p2wpkh")), + p2wpkh: SeriesTree_Indexes_Addr_P2wpkh::new( + client.clone(), + format!("{base_path}_p2wpkh"), + ), p2wsh: SeriesTree_Indexes_Addr_P2wsh::new(client.clone(), format!("{base_path}_p2wsh")), p2a: SeriesTree_Indexes_Addr_P2a::new(client.clone(), format!("{base_path}_p2a")), p2ms: SeriesTree_Indexes_Addr_P2ms::new(client.clone(), format!("{base_path}_p2ms")), empty: SeriesTree_Indexes_Addr_Empty::new(client.clone(), format!("{base_path}_empty")), - unknown: SeriesTree_Indexes_Addr_Unknown::new(client.clone(), format!("{base_path}_unknown")), - op_return: SeriesTree_Indexes_Addr_OpReturn::new(client.clone(), format!("{base_path}_op_return")), + unknown: SeriesTree_Indexes_Addr_Unknown::new( + client.clone(), + format!("{base_path}_unknown"), + ), + op_return: SeriesTree_Indexes_Addr_OpReturn::new( + client.clone(), + format!("{base_path}_op_return"), + ), } } } @@ -6892,13 +9365,28 @@ impl SeriesTree_Indicators { nvt: PpmRatioPattern3::new(client.clone(), "nvt".to_string()), gini: PercentPpmRatioPattern2::new(client.clone(), "gini".to_string()), rhodl_ratio: PpmRatioPattern3::new(client.clone(), "rhodl_ratio".to_string()), - thermo_cap_multiple: PpmRatioPattern3::new(client.clone(), "thermo_cap_multiple".to_string()), - coindays_destroyed_supply_adj: SeriesPattern1::new(client.clone(), "coindays_destroyed_supply_adj".to_string()), - coinyears_destroyed_supply_adj: SeriesPattern1::new(client.clone(), "coinyears_destroyed_supply_adj".to_string()), - dormancy: SeriesTree_Indicators_Dormancy::new(client.clone(), format!("{base_path}_dormancy")), + thermo_cap_multiple: PpmRatioPattern3::new( + client.clone(), + "thermo_cap_multiple".to_string(), + ), + coindays_destroyed_supply_adj: SeriesPattern1::new( + client.clone(), + "coindays_destroyed_supply_adj".to_string(), + ), + coinyears_destroyed_supply_adj: SeriesPattern1::new( + client.clone(), + "coinyears_destroyed_supply_adj".to_string(), + ), + dormancy: SeriesTree_Indicators_Dormancy::new( + client.clone(), + format!("{base_path}_dormancy"), + ), stock_to_flow: SeriesPattern1::new(client.clone(), "stock_to_flow".to_string()), seller_exhaustion: SeriesPattern1::new(client.clone(), "seller_exhaustion".to_string()), - rarity_meter: SeriesTree_Indicators_RarityMeter::new(client.clone(), format!("{base_path}_rarity_meter")), + rarity_meter: SeriesTree_Indicators_RarityMeter::new( + client.clone(), + format!("{base_path}_rarity_meter"), + ), } } } @@ -6939,21 +9427,36 @@ impl SeriesTree_Indicators_RarityMeter { /// Series tree node. pub struct SeriesTree_Indicators_RarityMeter_Components { - pub realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub capitalized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub sth_realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub sth_capitalized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub lth_realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub lth_capitalized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub over_6m_realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub over_4m_realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub under_4m_realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub under_6m_realized_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub vaulted_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub active_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub true_market_mean_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub cointime_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, - pub coinflow_price: Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub capitalized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub sth_realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub sth_capitalized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub lth_realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub lth_capitalized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub over_6m_realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub over_4m_realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub under_4m_realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub under_6m_realized_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub vaulted_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub active_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub true_market_mean_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub cointime_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, + pub coinflow_price: + Pct0Pct1Pct10Pct2Pct20Pct30Pct40Pct5Pct50Pct60Pct70Pct80Pct90Pct95Pct98Pct99Pattern, } impl SeriesTree_Indicators_RarityMeter_Components { @@ -7008,12 +9511,27 @@ pub struct SeriesTree_Investing_Period { impl SeriesTree_Investing_Period { pub fn new(client: Arc, base_path: String) -> Self { Self { - dca_stack: _10y1m1w1y2y3m3y4y5y6m6y8yPattern3::new(client.clone(), "dca_stack".to_string()), - dca_cost_basis: SeriesTree_Investing_Period_DcaCostBasis::new(client.clone(), format!("{base_path}_dca_cost_basis")), - dca_return: _10y1m1w1y2y3m3y4y5y6m6y8yPattern2::new(client.clone(), "dca_return".to_string()), + dca_stack: _10y1m1w1y2y3m3y4y5y6m6y8yPattern3::new( + client.clone(), + "dca_stack".to_string(), + ), + dca_cost_basis: SeriesTree_Investing_Period_DcaCostBasis::new( + client.clone(), + format!("{base_path}_dca_cost_basis"), + ), + dca_return: _10y1m1w1y2y3m3y4y5y6m6y8yPattern2::new( + client.clone(), + "dca_return".to_string(), + ), dca_cagr: _10y2y3y4y5y6y8yPattern::new(client.clone(), "dca_cagr".to_string()), - lump_sum_stack: _10y1m1w1y2y3m3y4y5y6m6y8yPattern3::new(client.clone(), "lump_sum_stack".to_string()), - lump_sum_return: _10y1m1w1y2y3m3y4y5y6m6y8yPattern2::new(client.clone(), "lump_sum_return".to_string()), + lump_sum_stack: _10y1m1w1y2y3m3y4y5y6m6y8yPattern3::new( + client.clone(), + "lump_sum_stack".to_string(), + ), + lump_sum_return: _10y1m1w1y2y3m3y4y5y6m6y8yPattern2::new( + client.clone(), + "lump_sum_return".to_string(), + ), } } } @@ -7063,9 +9581,18 @@ pub struct SeriesTree_Investing_Class { impl SeriesTree_Investing_Class { pub fn new(client: Arc, base_path: String) -> Self { Self { - dca_stack: SeriesTree_Investing_Class_DcaStack::new(client.clone(), format!("{base_path}_dca_stack")), - dca_cost_basis: SeriesTree_Investing_Class_DcaCostBasis::new(client.clone(), format!("{base_path}_dca_cost_basis")), - dca_return: SeriesTree_Investing_Class_DcaReturn::new(client.clone(), format!("{base_path}_dca_return")), + dca_stack: SeriesTree_Investing_Class_DcaStack::new( + client.clone(), + format!("{base_path}_dca_stack"), + ), + dca_cost_basis: SeriesTree_Investing_Class_DcaCostBasis::new( + client.clone(), + format!("{base_path}_dca_cost_basis"), + ), + dca_return: SeriesTree_Investing_Class_DcaReturn::new( + client.clone(), + format!("{base_path}_dca_return"), + ), } } } @@ -7089,18 +9616,54 @@ pub struct SeriesTree_Investing_Class_DcaStack { impl SeriesTree_Investing_Class_DcaStack { pub fn new(client: Arc, base_path: String) -> Self { Self { - from_2015: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2015".to_string()), - from_2016: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2016".to_string()), - from_2017: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2017".to_string()), - from_2018: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2018".to_string()), - from_2019: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2019".to_string()), - from_2020: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2020".to_string()), - from_2021: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2021".to_string()), - from_2022: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2022".to_string()), - from_2023: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2023".to_string()), - from_2024: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2024".to_string()), - from_2025: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2025".to_string()), - from_2026: BtcCentsSatsUsdPattern::new(client.clone(), "dca_stack_from_2026".to_string()), + from_2015: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2015".to_string(), + ), + from_2016: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2016".to_string(), + ), + from_2017: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2017".to_string(), + ), + from_2018: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2018".to_string(), + ), + from_2019: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2019".to_string(), + ), + from_2020: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2020".to_string(), + ), + from_2021: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2021".to_string(), + ), + from_2022: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2022".to_string(), + ), + from_2023: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2023".to_string(), + ), + from_2024: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2024".to_string(), + ), + from_2025: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2025".to_string(), + ), + from_2026: BtcCentsSatsUsdPattern::new( + client.clone(), + "dca_stack_from_2026".to_string(), + ), } } } @@ -7124,18 +9687,54 @@ pub struct SeriesTree_Investing_Class_DcaCostBasis { impl SeriesTree_Investing_Class_DcaCostBasis { pub fn new(client: Arc, base_path: String) -> Self { Self { - from_2015: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2015".to_string()), - from_2016: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2016".to_string()), - from_2017: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2017".to_string()), - from_2018: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2018".to_string()), - from_2019: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2019".to_string()), - from_2020: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2020".to_string()), - from_2021: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2021".to_string()), - from_2022: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2022".to_string()), - from_2023: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2023".to_string()), - from_2024: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2024".to_string()), - from_2025: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2025".to_string()), - from_2026: CentsSatsUsdPattern::new(client.clone(), "dca_cost_basis_from_2026".to_string()), + from_2015: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2015".to_string(), + ), + from_2016: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2016".to_string(), + ), + from_2017: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2017".to_string(), + ), + from_2018: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2018".to_string(), + ), + from_2019: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2019".to_string(), + ), + from_2020: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2020".to_string(), + ), + from_2021: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2021".to_string(), + ), + from_2022: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2022".to_string(), + ), + from_2023: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2023".to_string(), + ), + from_2024: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2024".to_string(), + ), + from_2025: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2025".to_string(), + ), + from_2026: CentsSatsUsdPattern::new( + client.clone(), + "dca_cost_basis_from_2026".to_string(), + ), } } } @@ -7159,18 +9758,54 @@ pub struct SeriesTree_Investing_Class_DcaReturn { impl SeriesTree_Investing_Class_DcaReturn { pub fn new(client: Arc, base_path: String) -> Self { Self { - from_2015: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2015".to_string()), - from_2016: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2016".to_string()), - from_2017: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2017".to_string()), - from_2018: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2018".to_string()), - from_2019: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2019".to_string()), - from_2020: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2020".to_string()), - from_2021: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2021".to_string()), - from_2022: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2022".to_string()), - from_2023: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2023".to_string()), - from_2024: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2024".to_string()), - from_2025: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2025".to_string()), - from_2026: PercentPpmRatioPattern::new(client.clone(), "dca_return_from_2026".to_string()), + from_2015: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2015".to_string(), + ), + from_2016: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2016".to_string(), + ), + from_2017: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2017".to_string(), + ), + from_2018: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2018".to_string(), + ), + from_2019: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2019".to_string(), + ), + from_2020: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2020".to_string(), + ), + from_2021: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2021".to_string(), + ), + from_2022: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2022".to_string(), + ), + from_2023: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2023".to_string(), + ), + from_2024: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2024".to_string(), + ), + from_2025: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2025".to_string(), + ), + from_2026: PercentPpmRatioPattern::new( + client.clone(), + "dca_return_from_2026".to_string(), + ), } } } @@ -7190,12 +9825,21 @@ impl SeriesTree_Market { pub fn new(client: Arc, base_path: String) -> Self { Self { ath: SeriesTree_Market_Ath::new(client.clone(), format!("{base_path}_ath")), - lookback: SeriesTree_Market_Lookback::new(client.clone(), format!("{base_path}_lookback")), + lookback: SeriesTree_Market_Lookback::new( + client.clone(), + format!("{base_path}_lookback"), + ), returns: SeriesTree_Market_Returns::new(client.clone(), format!("{base_path}_returns")), volatility: _1m1w1y24hPattern::new(client.clone(), "price_volatility".to_string()), range: SeriesTree_Market_Range::new(client.clone(), format!("{base_path}_range")), - moving_average: SeriesTree_Market_MovingAverage::new(client.clone(), format!("{base_path}_moving_average")), - technical: SeriesTree_Market_Technical::new(client.clone(), format!("{base_path}_technical")), + moving_average: SeriesTree_Market_MovingAverage::new( + client.clone(), + format!("{base_path}_moving_average"), + ), + technical: SeriesTree_Market_Technical::new( + client.clone(), + format!("{base_path}_technical"), + ), } } } @@ -7217,8 +9861,14 @@ impl SeriesTree_Market_Ath { drawdown: PercentPpmRatioPattern3::new(client.clone(), "price_drawdown".to_string()), days_since: SeriesPattern1::new(client.clone(), "days_since_price_ath".to_string()), years_since: SeriesPattern1::new(client.clone(), "years_since_price_ath".to_string()), - max_days_between: SeriesPattern1::new(client.clone(), "max_days_between_price_ath".to_string()), - max_years_between: SeriesPattern1::new(client.clone(), "max_years_between_price_ath".to_string()), + max_days_between: SeriesPattern1::new( + client.clone(), + "max_days_between_price_ath".to_string(), + ), + max_years_between: SeriesPattern1::new( + client.clone(), + "max_years_between_price_ath".to_string(), + ), } } } @@ -7270,9 +9920,15 @@ pub struct SeriesTree_Market_Returns { impl SeriesTree_Market_Returns { pub fn new(client: Arc, base_path: String) -> Self { Self { - periods: SeriesTree_Market_Returns_Periods::new(client.clone(), format!("{base_path}_periods")), + periods: SeriesTree_Market_Returns_Periods::new( + client.clone(), + format!("{base_path}_periods"), + ), cagr: _10y2y3y4y5y6y8yPattern::new(client.clone(), "price_cagr".to_string()), - sd_24h: SeriesTree_Market_Returns_Sd24h::new(client.clone(), format!("{base_path}_sd_24h")), + sd_24h: SeriesTree_Market_Returns_Sd24h::new( + client.clone(), + format!("{base_path}_sd_24h"), + ), } } } @@ -7325,7 +9981,10 @@ pub struct SeriesTree_Market_Returns_Sd24h { impl SeriesTree_Market_Returns_Sd24h { pub fn new(client: Arc, base_path: String) -> Self { Self { - _24h: SeriesTree_Market_Returns_Sd24h_24h::new(client.clone(), format!("{base_path}_24h")), + _24h: SeriesTree_Market_Returns_Sd24h_24h::new( + client.clone(), + format!("{base_path}_24h"), + ), _1w: SeriesTree_Market_Returns_Sd24h_1w::new(client.clone(), format!("{base_path}_1w")), _1m: SeriesTree_Market_Returns_Sd24h_1m::new(client.clone(), format!("{base_path}_1m")), _1y: SeriesTree_Market_Returns_Sd24h_1y::new(client.clone(), format!("{base_path}_1y")), @@ -7408,8 +10067,14 @@ impl SeriesTree_Market_Range { min: _1m1w1y2wPattern::new(client.clone(), "price_min".to_string()), max: _1m1w1y2wPattern::new(client.clone(), "price_max".to_string()), true_range: SeriesPattern1::new(client.clone(), "price_true_range".to_string()), - true_range_sum_2w: SeriesPattern1::new(client.clone(), "price_true_range_sum_2w".to_string()), - choppiness_index_2w: PercentPpmRatioPattern2::new(client.clone(), "price_choppiness_index_2w".to_string()), + true_range_sum_2w: SeriesPattern1::new( + client.clone(), + "price_true_range_sum_2w".to_string(), + ), + choppiness_index_2w: PercentPpmRatioPattern2::new( + client.clone(), + "price_choppiness_index_2w".to_string(), + ), } } } @@ -7423,8 +10088,14 @@ pub struct SeriesTree_Market_MovingAverage { impl SeriesTree_Market_MovingAverage { pub fn new(client: Arc, base_path: String) -> Self { Self { - sma: SeriesTree_Market_MovingAverage_Sma::new(client.clone(), format!("{base_path}_sma")), - ema: SeriesTree_Market_MovingAverage_Ema::new(client.clone(), format!("{base_path}_ema")), + sma: SeriesTree_Market_MovingAverage_Sma::new( + client.clone(), + format!("{base_path}_sma"), + ), + ema: SeriesTree_Market_MovingAverage_Ema::new( + client.clone(), + format!("{base_path}_ema"), + ), } } } @@ -7462,8 +10133,14 @@ impl SeriesTree_Market_MovingAverage_Sma { _89d: CentsPpmRatioSatsUsdPattern::new(client.clone(), "price_sma_89d".to_string()), _111d: CentsPpmRatioSatsUsdPattern::new(client.clone(), "price_sma_111d".to_string()), _144d: CentsPpmRatioSatsUsdPattern::new(client.clone(), "price_sma_144d".to_string()), - _200d: SeriesTree_Market_MovingAverage_Sma_200d::new(client.clone(), format!("{base_path}_200d")), - _350d: SeriesTree_Market_MovingAverage_Sma_350d::new(client.clone(), format!("{base_path}_350d")), + _200d: SeriesTree_Market_MovingAverage_Sma_200d::new( + client.clone(), + format!("{base_path}_200d"), + ), + _350d: SeriesTree_Market_MovingAverage_Sma_350d::new( + client.clone(), + format!("{base_path}_350d"), + ), _1y: CentsPpmRatioSatsUsdPattern::new(client.clone(), "price_sma_1y".to_string()), _2y: CentsPpmRatioSatsUsdPattern::new(client.clone(), "price_sma_2y".to_string()), _200w: CentsPpmRatioSatsUsdPattern::new(client.clone(), "price_sma_200w".to_string()), @@ -7575,7 +10252,10 @@ impl SeriesTree_Market_Technical { Self { rsi: SeriesTree_Market_Technical_Rsi::new(client.clone(), format!("{base_path}_rsi")), pi_cycle: PpmRatioPattern2::new(client.clone(), "pi_cycle".to_string()), - macd: SeriesTree_Market_Technical_Macd::new(client.clone(), format!("{base_path}_macd")), + macd: SeriesTree_Market_Technical_Macd::new( + client.clone(), + format!("{base_path}_macd"), + ), } } } @@ -7607,9 +10287,18 @@ pub struct SeriesTree_Market_Technical_Macd { impl SeriesTree_Market_Technical_Macd { pub fn new(client: Arc, base_path: String) -> Self { Self { - _24h: SeriesTree_Market_Technical_Macd_24h::new(client.clone(), format!("{base_path}_24h")), - _1w: SeriesTree_Market_Technical_Macd_1w::new(client.clone(), format!("{base_path}_1w")), - _1m: SeriesTree_Market_Technical_Macd_1m::new(client.clone(), format!("{base_path}_1m")), + _24h: SeriesTree_Market_Technical_Macd_24h::new( + client.clone(), + format!("{base_path}_24h"), + ), + _1w: SeriesTree_Market_Technical_Macd_1w::new( + client.clone(), + format!("{base_path}_1w"), + ), + _1m: SeriesTree_Market_Technical_Macd_1m::new( + client.clone(), + format!("{base_path}_1m"), + ), } } } @@ -7730,16 +10419,28 @@ impl SeriesTree_Pools_Major { btcguild: BlocksDominanceRewardsPattern::new(client.clone(), "btcguild".to_string()), eligius: BlocksDominanceRewardsPattern::new(client.clone(), "eligius".to_string()), f2pool: BlocksDominanceRewardsPattern::new(client.clone(), "f2pool".to_string()), - braiinspool: BlocksDominanceRewardsPattern::new(client.clone(), "braiinspool".to_string()), + braiinspool: BlocksDominanceRewardsPattern::new( + client.clone(), + "braiinspool".to_string(), + ), antpool: BlocksDominanceRewardsPattern::new(client.clone(), "antpool".to_string()), btcc: BlocksDominanceRewardsPattern::new(client.clone(), "btcc".to_string()), bwpool: BlocksDominanceRewardsPattern::new(client.clone(), "bwpool".to_string()), bitfury: BlocksDominanceRewardsPattern::new(client.clone(), "bitfury".to_string()), viabtc: BlocksDominanceRewardsPattern::new(client.clone(), "viabtc".to_string()), poolin: BlocksDominanceRewardsPattern::new(client.clone(), "poolin".to_string()), - spiderpool: BlocksDominanceRewardsPattern::new(client.clone(), "spiderpool".to_string()), - binancepool: BlocksDominanceRewardsPattern::new(client.clone(), "binancepool".to_string()), - foundryusa: BlocksDominanceRewardsPattern::new(client.clone(), "foundryusa".to_string()), + spiderpool: BlocksDominanceRewardsPattern::new( + client.clone(), + "spiderpool".to_string(), + ), + binancepool: BlocksDominanceRewardsPattern::new( + client.clone(), + "binancepool".to_string(), + ), + foundryusa: BlocksDominanceRewardsPattern::new( + client.clone(), + "foundryusa".to_string(), + ), sbicrypto: BlocksDominanceRewardsPattern::new(client.clone(), "sbicrypto".to_string()), marapool: BlocksDominanceRewardsPattern::new(client.clone(), "marapool".to_string()), secpool: BlocksDominanceRewardsPattern::new(client.clone(), "secpool".to_string()), @@ -7944,7 +10645,10 @@ impl SeriesTree_Pools_Minor { ckpool: BlocksDominancePattern::new(client.clone(), "ckpool".to_string()), nicehash: BlocksDominancePattern::new(client.clone(), "nicehash".to_string()), bitclub: BlocksDominancePattern::new(client.clone(), "bitclub".to_string()), - bitcoinaffiliatenetwork: BlocksDominancePattern::new(client.clone(), "bitcoinaffiliatenetwork".to_string()), + bitcoinaffiliatenetwork: BlocksDominancePattern::new( + client.clone(), + "bitcoinaffiliatenetwork".to_string(), + ), exxbw: BlocksDominancePattern::new(client.clone(), "exxbw".to_string()), bitsolo: BlocksDominancePattern::new(client.clone(), "bitsolo".to_string()), twentyoneinc: BlocksDominancePattern::new(client.clone(), "twentyoneinc".to_string()), @@ -7967,7 +10671,10 @@ impl SeriesTree_Pools_Minor { dcexploration: BlocksDominancePattern::new(client.clone(), "dcexploration".to_string()), dcex: BlocksDominancePattern::new(client.clone(), "dcex".to_string()), btpool: BlocksDominancePattern::new(client.clone(), "btpool".to_string()), - fiftyeightcoin: BlocksDominancePattern::new(client.clone(), "fiftyeightcoin".to_string()), + fiftyeightcoin: BlocksDominancePattern::new( + client.clone(), + "fiftyeightcoin".to_string(), + ), bitcoinindia: BlocksDominancePattern::new(client.clone(), "bitcoinindia".to_string()), shawnp0wers: BlocksDominancePattern::new(client.clone(), "shawnp0wers".to_string()), phashio: BlocksDominancePattern::new(client.clone(), "phashio".to_string()), @@ -7980,8 +10687,14 @@ impl SeriesTree_Pools_Minor { rawpool: BlocksDominancePattern::new(client.clone(), "rawpool".to_string()), haominer: BlocksDominancePattern::new(client.clone(), "haominer".to_string()), helix: BlocksDominancePattern::new(client.clone(), "helix".to_string()), - bitcoinukraine: BlocksDominancePattern::new(client.clone(), "bitcoinukraine".to_string()), - secretsuperstar: BlocksDominancePattern::new(client.clone(), "secretsuperstar".to_string()), + bitcoinukraine: BlocksDominancePattern::new( + client.clone(), + "bitcoinukraine".to_string(), + ), + secretsuperstar: BlocksDominancePattern::new( + client.clone(), + "secretsuperstar".to_string(), + ), tigerpoolnet: BlocksDominancePattern::new(client.clone(), "tigerpoolnet".to_string()), sigmapoolcom: BlocksDominancePattern::new(client.clone(), "sigmapoolcom".to_string()), okpooltop: BlocksDominancePattern::new(client.clone(), "okpooltop".to_string()), @@ -7998,25 +10711,40 @@ impl SeriesTree_Pools_Minor { arkpool: BlocksDominancePattern::new(client.clone(), "arkpool".to_string()), purebtccom: BlocksDominancePattern::new(client.clone(), "purebtccom".to_string()), kucoinpool: BlocksDominancePattern::new(client.clone(), "kucoinpool".to_string()), - entrustcharitypool: BlocksDominancePattern::new(client.clone(), "entrustcharitypool".to_string()), + entrustcharitypool: BlocksDominancePattern::new( + client.clone(), + "entrustcharitypool".to_string(), + ), okminer: BlocksDominancePattern::new(client.clone(), "okminer".to_string()), titan: BlocksDominancePattern::new(client.clone(), "titan".to_string()), pegapool: BlocksDominancePattern::new(client.clone(), "pegapool".to_string()), btcnuggets: BlocksDominancePattern::new(client.clone(), "btcnuggets".to_string()), cloudhashing: BlocksDominancePattern::new(client.clone(), "cloudhashing".to_string()), - digitalxmintsy: BlocksDominancePattern::new(client.clone(), "digitalxmintsy".to_string()), + digitalxmintsy: BlocksDominancePattern::new( + client.clone(), + "digitalxmintsy".to_string(), + ), telco214: BlocksDominancePattern::new(client.clone(), "telco214".to_string()), btcpoolparty: BlocksDominancePattern::new(client.clone(), "btcpoolparty".to_string()), multipool: BlocksDominancePattern::new(client.clone(), "multipool".to_string()), - transactioncoinmining: BlocksDominancePattern::new(client.clone(), "transactioncoinmining".to_string()), + transactioncoinmining: BlocksDominancePattern::new( + client.clone(), + "transactioncoinmining".to_string(), + ), btcdig: BlocksDominancePattern::new(client.clone(), "btcdig".to_string()), - trickysbtcpool: BlocksDominancePattern::new(client.clone(), "trickysbtcpool".to_string()), + trickysbtcpool: BlocksDominancePattern::new( + client.clone(), + "trickysbtcpool".to_string(), + ), btcmp: BlocksDominancePattern::new(client.clone(), "btcmp".to_string()), eobot: BlocksDominancePattern::new(client.clone(), "eobot".to_string()), unomp: BlocksDominancePattern::new(client.clone(), "unomp".to_string()), patels: BlocksDominancePattern::new(client.clone(), "patels".to_string()), gogreenlight: BlocksDominancePattern::new(client.clone(), "gogreenlight".to_string()), - bitcoinindiapool: BlocksDominancePattern::new(client.clone(), "bitcoinindiapool".to_string()), + bitcoinindiapool: BlocksDominancePattern::new( + client.clone(), + "bitcoinindiapool".to_string(), + ), ekanembtc: BlocksDominancePattern::new(client.clone(), "ekanembtc".to_string()), canoe: BlocksDominancePattern::new(client.clone(), "canoe".to_string()), tiger: BlocksDominancePattern::new(client.clone(), "tiger".to_string()), @@ -8024,8 +10752,14 @@ impl SeriesTree_Pools_Minor { zulupool: BlocksDominancePattern::new(client.clone(), "zulupool".to_string()), wiz: BlocksDominancePattern::new(client.clone(), "wiz".to_string()), wk057: BlocksDominancePattern::new(client.clone(), "wk057".to_string()), - futurebitapollosolo: BlocksDominancePattern::new(client.clone(), "futurebitapollosolo".to_string()), - carbonnegative: BlocksDominancePattern::new(client.clone(), "carbonnegative".to_string()), + futurebitapollosolo: BlocksDominancePattern::new( + client.clone(), + "futurebitapollosolo".to_string(), + ), + carbonnegative: BlocksDominancePattern::new( + client.clone(), + "carbonnegative".to_string(), + ), portlandhodl: BlocksDominancePattern::new(client.clone(), "portlandhodl".to_string()), phoenix: BlocksDominancePattern::new(client.clone(), "phoenix".to_string()), neopool: BlocksDominancePattern::new(client.clone(), "neopool".to_string()), @@ -8134,13 +10868,28 @@ impl SeriesTree_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { state: SeriesPattern18::new(client.clone(), "supply_state".to_string()), - circulating: BtcCentsSatsUsdPattern::new(client.clone(), "circulating_supply".to_string()), + circulating: BtcCentsSatsUsdPattern::new( + client.clone(), + "circulating_supply".to_string(), + ), burned: BlockCumulativePattern::new(client.clone(), "unspendable_supply".to_string()), - inflation_rate: PercentPpmRatioPattern::new(client.clone(), "inflation_rate".to_string()), - velocity: SeriesTree_Supply_Velocity::new(client.clone(), format!("{base_path}_velocity")), + inflation_rate: PercentPpmRatioPattern::new( + client.clone(), + "inflation_rate".to_string(), + ), + velocity: SeriesTree_Supply_Velocity::new( + client.clone(), + format!("{base_path}_velocity"), + ), market_cap: CentsDeltaUsdPattern::new(client.clone(), "market_cap".to_string()), - market_minus_realized_cap_growth_rate: _1m1w1y24hPattern::new(client.clone(), "market_minus_realized_cap_growth_rate".to_string()), - hodled_or_lost: BtcCentsSatsUsdPattern::new(client.clone(), "hodled_or_lost_supply".to_string()), + market_minus_realized_cap_growth_rate: _1m1w1y24hPattern::new( + client.clone(), + "market_minus_realized_cap_growth_rate".to_string(), + ), + hodled_or_lost: BtcCentsSatsUsdPattern::new( + client.clone(), + "hodled_or_lost_supply".to_string(), + ), } } } @@ -8200,18 +10949,42 @@ impl SeriesTree_Cohorts_Utxo { all: SeriesTree_Cohorts_Utxo_All::new(client.clone(), format!("{base_path}_all")), sth: SeriesTree_Cohorts_Utxo_Sth::new(client.clone(), format!("{base_path}_sth")), lth: SeriesTree_Cohorts_Utxo_Lth::new(client.clone(), format!("{base_path}_lth")), - age_range: SeriesTree_Cohorts_Utxo_AgeRange::new(client.clone(), format!("{base_path}_age_range")), - under_age: SeriesTree_Cohorts_Utxo_UnderAge::new(client.clone(), format!("{base_path}_under_age")), - over_age: SeriesTree_Cohorts_Utxo_OverAge::new(client.clone(), format!("{base_path}_over_age")), + age_range: SeriesTree_Cohorts_Utxo_AgeRange::new( + client.clone(), + format!("{base_path}_age_range"), + ), + under_age: SeriesTree_Cohorts_Utxo_UnderAge::new( + client.clone(), + format!("{base_path}_under_age"), + ), + over_age: SeriesTree_Cohorts_Utxo_OverAge::new( + client.clone(), + format!("{base_path}_over_age"), + ), epoch: SeriesTree_Cohorts_Utxo_Epoch::new(client.clone(), format!("{base_path}_epoch")), class: SeriesTree_Cohorts_Utxo_Class::new(client.clone(), format!("{base_path}_class")), entry: SeriesTree_Cohorts_Utxo_Entry::new(client.clone(), format!("{base_path}_entry")), - over_amount: SeriesTree_Cohorts_Utxo_OverAmount::new(client.clone(), format!("{base_path}_over_amount")), - amount_range: SeriesTree_Cohorts_Utxo_AmountRange::new(client.clone(), format!("{base_path}_amount_range")), - under_amount: SeriesTree_Cohorts_Utxo_UnderAmount::new(client.clone(), format!("{base_path}_under_amount")), + over_amount: SeriesTree_Cohorts_Utxo_OverAmount::new( + client.clone(), + format!("{base_path}_over_amount"), + ), + amount_range: SeriesTree_Cohorts_Utxo_AmountRange::new( + client.clone(), + format!("{base_path}_amount_range"), + ), + under_amount: SeriesTree_Cohorts_Utxo_UnderAmount::new( + client.clone(), + format!("{base_path}_under_amount"), + ), type_: SeriesTree_Cohorts_Utxo_Type::new(client.clone(), format!("{base_path}_type")), - profitability: SeriesTree_Cohorts_Utxo_Profitability::new(client.clone(), format!("{base_path}_profitability")), - matured: SeriesTree_Cohorts_Utxo_Matured::new(client.clone(), format!("{base_path}_matured")), + profitability: SeriesTree_Cohorts_Utxo_Profitability::new( + client.clone(), + format!("{base_path}_profitability"), + ), + matured: SeriesTree_Cohorts_Utxo_Matured::new( + client.clone(), + format!("{base_path}_matured"), + ), } } } @@ -8231,11 +11004,26 @@ impl SeriesTree_Cohorts_Utxo_All { pub fn new(client: Arc, base_path: String) -> Self { Self { supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "supply".to_string()), - outputs: SeriesTree_Cohorts_Utxo_All_Outputs::new(client.clone(), format!("{base_path}_outputs")), - activity: SeriesTree_Cohorts_Utxo_All_Activity::new(client.clone(), format!("{base_path}_activity")), - realized: SeriesTree_Cohorts_Utxo_All_Realized::new(client.clone(), format!("{base_path}_realized")), - cost_basis: SeriesTree_Cohorts_Utxo_All_CostBasis::new(client.clone(), format!("{base_path}_cost_basis")), - unrealized: SeriesTree_Cohorts_Utxo_All_Unrealized::new(client.clone(), format!("{base_path}_unrealized")), + outputs: SeriesTree_Cohorts_Utxo_All_Outputs::new( + client.clone(), + format!("{base_path}_outputs"), + ), + activity: SeriesTree_Cohorts_Utxo_All_Activity::new( + client.clone(), + format!("{base_path}_activity"), + ), + realized: SeriesTree_Cohorts_Utxo_All_Realized::new( + client.clone(), + format!("{base_path}_realized"), + ), + cost_basis: SeriesTree_Cohorts_Utxo_All_CostBasis::new( + client.clone(), + format!("{base_path}_cost_basis"), + ), + unrealized: SeriesTree_Cohorts_Utxo_All_Unrealized::new( + client.clone(), + format!("{base_path}_unrealized"), + ), invested_capital: InPattern::new(client.clone(), "invested_capital_in".to_string()), } } @@ -8251,7 +11039,10 @@ impl SeriesTree_Cohorts_Utxo_All_Outputs { pub fn new(client: Arc, base_path: String) -> Self { Self { unspent_count: BaseDeltaPattern::new(client.clone(), "utxo_count".to_string()), - spent_count: AverageBlockCumulativeSumPattern::new(client.clone(), "spent_utxo_count".to_string()), + spent_count: AverageBlockCumulativeSumPattern::new( + client.clone(), + "spent_utxo_count".to_string(), + ), } } } @@ -8267,9 +11058,18 @@ pub struct SeriesTree_Cohorts_Utxo_All_Activity { impl SeriesTree_Cohorts_Utxo_All_Activity { pub fn new(client: Arc, base_path: String) -> Self { Self { - transfer_volume: AverageBlockCumulativeInSumPattern::new(client.clone(), "transfer_volume".to_string()), - coindays_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), "coindays_destroyed".to_string()), - coinyears_destroyed: SeriesPattern1::new(client.clone(), "coinyears_destroyed".to_string()), + transfer_volume: AverageBlockCumulativeInSumPattern::new( + client.clone(), + "transfer_volume".to_string(), + ), + coindays_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + "coindays_destroyed".to_string(), + ), + coinyears_destroyed: SeriesPattern1::new( + client.clone(), + "coinyears_destroyed".to_string(), + ), dormancy: _1m1w1y24hPattern::new(client.clone(), "dormancy".to_string()), } } @@ -8296,16 +11096,34 @@ impl SeriesTree_Cohorts_Utxo_All_Realized { Self { cap: CentsDeltaToUsdPattern::new(client.clone(), "realized_cap".to_string()), profit: BlockCumulativeSumPattern::new(client.clone(), "realized_profit".to_string()), - loss: BlockCumulativeNegativeSumPattern::new(client.clone(), "realized_loss".to_string()), + loss: BlockCumulativeNegativeSumPattern::new( + client.clone(), + "realized_loss".to_string(), + ), price: CentsPpmRatioSatsUsdPattern::new(client.clone(), "realized_price".to_string()), mvrv: SeriesPattern1::new(client.clone(), "mvrv".to_string()), net_pnl: BlockChangeCumulativeDeltaSumPattern::new(client.clone(), "net".to_string()), - sopr: SeriesTree_Cohorts_Utxo_All_Realized_Sopr::new(client.clone(), format!("{base_path}_sopr")), - gross_pnl: BlockCumulativeSumPattern::new(client.clone(), "realized_gross_pnl".to_string()), - sell_side_risk_ratio: _1m1w1y24hPattern8::new(client.clone(), "sell_side_risk_ratio".to_string()), - peak_regret: BlockCumulativeSumPattern::new(client.clone(), "realized_peak_regret".to_string()), + sopr: SeriesTree_Cohorts_Utxo_All_Realized_Sopr::new( + client.clone(), + format!("{base_path}_sopr"), + ), + gross_pnl: BlockCumulativeSumPattern::new( + client.clone(), + "realized_gross_pnl".to_string(), + ), + sell_side_risk_ratio: _1m1w1y24hPattern8::new( + client.clone(), + "sell_side_risk_ratio".to_string(), + ), + peak_regret: BlockCumulativeSumPattern::new( + client.clone(), + "realized_peak_regret".to_string(), + ), capitalized: PricePattern::new(client.clone(), "capitalized_price".to_string()), - profit_to_loss_ratio: _1m1w1y24hPattern::new(client.clone(), "realized_profit_to_loss_ratio".to_string()), + profit_to_loss_ratio: _1m1w1y24hPattern::new( + client.clone(), + "realized_profit_to_loss_ratio".to_string(), + ), } } } @@ -8320,9 +11138,15 @@ pub struct SeriesTree_Cohorts_Utxo_All_Realized_Sopr { impl SeriesTree_Cohorts_Utxo_All_Realized_Sopr { pub fn new(client: Arc, base_path: String) -> Self { Self { - value_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), "value_destroyed".to_string()), + value_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + "value_destroyed".to_string(), + ), ratio: _1m1w1y24hPattern::new(client.clone(), "sopr".to_string()), - adjusted: SeriesTree_Cohorts_Utxo_All_Realized_Sopr_Adjusted::new(client.clone(), format!("{base_path}_adjusted")), + adjusted: SeriesTree_Cohorts_Utxo_All_Realized_Sopr_Adjusted::new( + client.clone(), + format!("{base_path}_adjusted"), + ), } } } @@ -8338,8 +11162,14 @@ impl SeriesTree_Cohorts_Utxo_All_Realized_Sopr_Adjusted { pub fn new(client: Arc, base_path: String) -> Self { Self { ratio: _1m1w1y24hPattern::new(client.clone(), "asopr".to_string()), - transfer_volume: AverageBlockCumulativeSumPattern::new(client.clone(), "adj_value_created".to_string()), - value_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), "adj_value_destroyed".to_string()), + transfer_volume: AverageBlockCumulativeSumPattern::new( + client.clone(), + "adj_value_created".to_string(), + ), + value_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + "adj_value_destroyed".to_string(), + ), } } } @@ -8386,14 +11216,32 @@ impl SeriesTree_Cohorts_Utxo_All_Unrealized { pub fn new(client: Arc, base_path: String) -> Self { Self { nupl: PpmRatioPattern::new(client.clone(), "nupl".to_string()), - profit: SeriesTree_Cohorts_Utxo_All_Unrealized_Profit::new(client.clone(), format!("{base_path}_profit")), - loss: SeriesTree_Cohorts_Utxo_All_Unrealized_Loss::new(client.clone(), format!("{base_path}_loss")), - net_pnl: SeriesTree_Cohorts_Utxo_All_Unrealized_NetPnl::new(client.clone(), format!("{base_path}_net_pnl")), + profit: SeriesTree_Cohorts_Utxo_All_Unrealized_Profit::new( + client.clone(), + format!("{base_path}_profit"), + ), + loss: SeriesTree_Cohorts_Utxo_All_Unrealized_Loss::new( + client.clone(), + format!("{base_path}_loss"), + ), + net_pnl: SeriesTree_Cohorts_Utxo_All_Unrealized_NetPnl::new( + client.clone(), + format!("{base_path}_net_pnl"), + ), gross_pnl: CentsUsdPattern3::new(client.clone(), "unrealized_gross_pnl".to_string()), invested_capital: InPattern2::new(client.clone(), "invested_capital_in".to_string()), - capitalized_cap_in_profit_raw: SeriesPattern18::new(client.clone(), "capitalized_cap_in_profit_raw".to_string()), - capitalized_cap_in_loss_raw: SeriesPattern18::new(client.clone(), "capitalized_cap_in_loss_raw".to_string()), - sentiment: SeriesTree_Cohorts_Utxo_All_Unrealized_Sentiment::new(client.clone(), format!("{base_path}_sentiment")), + capitalized_cap_in_profit_raw: SeriesPattern18::new( + client.clone(), + "capitalized_cap_in_profit_raw".to_string(), + ), + capitalized_cap_in_loss_raw: SeriesPattern18::new( + client.clone(), + "capitalized_cap_in_loss_raw".to_string(), + ), + sentiment: SeriesTree_Cohorts_Utxo_All_Unrealized_Sentiment::new( + client.clone(), + format!("{base_path}_sentiment"), + ), } } } @@ -8411,8 +11259,14 @@ impl SeriesTree_Cohorts_Utxo_All_Unrealized_Profit { Self { usd: SeriesPattern1::new(client.clone(), "unrealized_profit".to_string()), cents: SeriesPattern1::new(client.clone(), "unrealized_profit_cents".to_string()), - to_mcap: PercentPpmRatioPattern2::new(client.clone(), "unrealized_profit_to_mcap".to_string()), - to_own_gross_pnl: PercentPpmRatioPattern2::new(client.clone(), "unrealized_profit_to_own_gross_pnl".to_string()), + to_mcap: PercentPpmRatioPattern2::new( + client.clone(), + "unrealized_profit_to_mcap".to_string(), + ), + to_own_gross_pnl: PercentPpmRatioPattern2::new( + client.clone(), + "unrealized_profit_to_own_gross_pnl".to_string(), + ), } } } @@ -8432,8 +11286,14 @@ impl SeriesTree_Cohorts_Utxo_All_Unrealized_Loss { usd: SeriesPattern1::new(client.clone(), "unrealized_loss".to_string()), cents: SeriesPattern1::new(client.clone(), "unrealized_loss_cents".to_string()), negative: SeriesPattern1::new(client.clone(), "unrealized_loss_neg".to_string()), - to_mcap: PercentPpmRatioPattern2::new(client.clone(), "unrealized_loss_to_mcap".to_string()), - to_own_gross_pnl: PercentPpmRatioPattern2::new(client.clone(), "unrealized_loss_to_own_gross_pnl".to_string()), + to_mcap: PercentPpmRatioPattern2::new( + client.clone(), + "unrealized_loss_to_mcap".to_string(), + ), + to_own_gross_pnl: PercentPpmRatioPattern2::new( + client.clone(), + "unrealized_loss_to_own_gross_pnl".to_string(), + ), } } } @@ -8450,7 +11310,10 @@ impl SeriesTree_Cohorts_Utxo_All_Unrealized_NetPnl { Self { usd: SeriesPattern1::new(client.clone(), "net_unrealized_pnl".to_string()), cents: SeriesPattern1::new(client.clone(), "net_unrealized_pnl_cents".to_string()), - to_own_gross_pnl: PercentPpmRatioPattern3::new(client.clone(), "net_unrealized_pnl_to_own_gross_pnl".to_string()), + to_own_gross_pnl: PercentPpmRatioPattern3::new( + client.clone(), + "net_unrealized_pnl_to_own_gross_pnl".to_string(), + ), } } } @@ -8486,12 +11349,24 @@ pub struct SeriesTree_Cohorts_Utxo_Sth { impl SeriesTree_Cohorts_Utxo_Sth { pub fn new(client: Arc, base_path: String) -> Self { Self { - supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "sth_supply".to_string()), + supply: DeltaDominanceHalfInTotalPattern2::new( + client.clone(), + "sth_supply".to_string(), + ), outputs: SpentUnspentPattern::new(client.clone(), "sth".to_string()), - activity: CoindaysCoinyearsDormancyTransferPattern::new(client.clone(), "sth".to_string()), - realized: CapCapitalizedGrossLossMvrvNetPeakPriceProfitSellSoprPattern::new(client.clone(), "sth".to_string()), + activity: CoindaysCoinyearsDormancyTransferPattern::new( + client.clone(), + "sth".to_string(), + ), + realized: CapCapitalizedGrossLossMvrvNetPeakPriceProfitSellSoprPattern::new( + client.clone(), + "sth".to_string(), + ), cost_basis: InMaxMinPerSupplyPattern::new(client.clone(), "sth".to_string()), - unrealized: CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2::new(client.clone(), "sth".to_string()), + unrealized: CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2::new( + client.clone(), + "sth".to_string(), + ), invested_capital: InPattern::new(client.clone(), "sth_invested_capital_in".to_string()), } } @@ -8511,12 +11386,24 @@ pub struct SeriesTree_Cohorts_Utxo_Lth { impl SeriesTree_Cohorts_Utxo_Lth { pub fn new(client: Arc, base_path: String) -> Self { Self { - supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "lth_supply".to_string()), + supply: DeltaDominanceHalfInTotalPattern2::new( + client.clone(), + "lth_supply".to_string(), + ), outputs: SpentUnspentPattern::new(client.clone(), "lth".to_string()), - activity: CoindaysCoinyearsDormancyTransferPattern::new(client.clone(), "lth".to_string()), - realized: SeriesTree_Cohorts_Utxo_Lth_Realized::new(client.clone(), format!("{base_path}_realized")), + activity: CoindaysCoinyearsDormancyTransferPattern::new( + client.clone(), + "lth".to_string(), + ), + realized: SeriesTree_Cohorts_Utxo_Lth_Realized::new( + client.clone(), + format!("{base_path}_realized"), + ), cost_basis: InMaxMinPerSupplyPattern::new(client.clone(), "lth".to_string()), - unrealized: CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2::new(client.clone(), "lth".to_string()), + unrealized: CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2::new( + client.clone(), + "lth".to_string(), + ), invested_capital: InPattern::new(client.clone(), "lth_invested_capital_in".to_string()), } } @@ -8542,17 +11429,44 @@ impl SeriesTree_Cohorts_Utxo_Lth_Realized { pub fn new(client: Arc, base_path: String) -> Self { Self { cap: CentsDeltaToUsdPattern::new(client.clone(), "lth_realized_cap".to_string()), - profit: BlockCumulativeSumPattern::new(client.clone(), "lth_realized_profit".to_string()), - loss: BlockCumulativeNegativeSumPattern::new(client.clone(), "lth_realized_loss".to_string()), - price: CentsPpmRatioSatsUsdPattern::new(client.clone(), "lth_realized_price".to_string()), + profit: BlockCumulativeSumPattern::new( + client.clone(), + "lth_realized_profit".to_string(), + ), + loss: BlockCumulativeNegativeSumPattern::new( + client.clone(), + "lth_realized_loss".to_string(), + ), + price: CentsPpmRatioSatsUsdPattern::new( + client.clone(), + "lth_realized_price".to_string(), + ), mvrv: SeriesPattern1::new(client.clone(), "lth_mvrv".to_string()), - net_pnl: BlockChangeCumulativeDeltaSumPattern::new(client.clone(), "lth_net".to_string()), - sopr: SeriesTree_Cohorts_Utxo_Lth_Realized_Sopr::new(client.clone(), format!("{base_path}_sopr")), - gross_pnl: BlockCumulativeSumPattern::new(client.clone(), "lth_realized_gross_pnl".to_string()), - sell_side_risk_ratio: _1m1w1y24hPattern8::new(client.clone(), "lth_sell_side_risk_ratio".to_string()), - peak_regret: BlockCumulativeSumPattern::new(client.clone(), "lth_realized_peak_regret".to_string()), + net_pnl: BlockChangeCumulativeDeltaSumPattern::new( + client.clone(), + "lth_net".to_string(), + ), + sopr: SeriesTree_Cohorts_Utxo_Lth_Realized_Sopr::new( + client.clone(), + format!("{base_path}_sopr"), + ), + gross_pnl: BlockCumulativeSumPattern::new( + client.clone(), + "lth_realized_gross_pnl".to_string(), + ), + sell_side_risk_ratio: _1m1w1y24hPattern8::new( + client.clone(), + "lth_sell_side_risk_ratio".to_string(), + ), + peak_regret: BlockCumulativeSumPattern::new( + client.clone(), + "lth_realized_peak_regret".to_string(), + ), capitalized: PricePattern::new(client.clone(), "lth_capitalized_price".to_string()), - profit_to_loss_ratio: _1m1w1y24hPattern::new(client.clone(), "lth_realized_profit_to_loss_ratio".to_string()), + profit_to_loss_ratio: _1m1w1y24hPattern::new( + client.clone(), + "lth_realized_profit_to_loss_ratio".to_string(), + ), } } } @@ -8566,7 +11480,10 @@ pub struct SeriesTree_Cohorts_Utxo_Lth_Realized_Sopr { impl SeriesTree_Cohorts_Utxo_Lth_Realized_Sopr { pub fn new(client: Arc, base_path: String) -> Self { Self { - value_destroyed: AverageBlockCumulativeSumPattern::new(client.clone(), "lth_value_destroyed".to_string()), + value_destroyed: AverageBlockCumulativeSumPattern::new( + client.clone(), + "lth_value_destroyed".to_string(), + ), ratio: _1m1w1y24hPattern::new(client.clone(), "lth_sopr".to_string()), } } @@ -8602,29 +11519,98 @@ pub struct SeriesTree_Cohorts_Utxo_AgeRange { impl SeriesTree_Cohorts_Utxo_AgeRange { pub fn new(client: Arc, base_path: String) -> Self { Self { - under_1h: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_1h_old".to_string()), - _1h_to_1d: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_1h_to_1d_old".to_string()), - _1d_to_1w: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_1d_to_1w_old".to_string()), - _1w_to_1m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_1w_to_1m_old".to_string()), - _1m_to_2m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_1m_to_2m_old".to_string()), - _2m_to_3m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_2m_to_3m_old".to_string()), - _3m_to_4m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_3m_to_4m_old".to_string()), - _4m_to_5m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_4m_to_5m_old".to_string()), - _5m_to_6m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_5m_to_6m_old".to_string()), - _6m_to_9m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_6m_to_9m_old".to_string()), - _9m_to_1y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_9m_to_1y_old".to_string()), - _1y_to_18m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_1y_to_18m_old".to_string()), - _18m_to_2y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_18m_to_2y_old".to_string()), - _2y_to_3y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_2y_to_3y_old".to_string()), - _3y_to_4y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_3y_to_4y_old".to_string()), - _4y_to_5y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_4y_to_5y_old".to_string()), - _5y_to_6y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_5y_to_6y_old".to_string()), - _6y_to_7y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_6y_to_7y_old".to_string()), - _7y_to_8y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_7y_to_8y_old".to_string()), - _8y_to_10y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_8y_to_10y_old".to_string()), - _10y_to_12y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_10y_to_12y_old".to_string()), - _12y_to_15y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_12y_to_15y_old".to_string()), - over_15y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_15y_old".to_string()), + under_1h: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_1h_old".to_string(), + ), + _1h_to_1d: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_1h_to_1d_old".to_string(), + ), + _1d_to_1w: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_1d_to_1w_old".to_string(), + ), + _1w_to_1m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_1w_to_1m_old".to_string(), + ), + _1m_to_2m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_1m_to_2m_old".to_string(), + ), + _2m_to_3m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_2m_to_3m_old".to_string(), + ), + _3m_to_4m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_3m_to_4m_old".to_string(), + ), + _4m_to_5m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_4m_to_5m_old".to_string(), + ), + _5m_to_6m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_5m_to_6m_old".to_string(), + ), + _6m_to_9m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_6m_to_9m_old".to_string(), + ), + _9m_to_1y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_9m_to_1y_old".to_string(), + ), + _1y_to_18m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_1y_to_18m_old".to_string(), + ), + _18m_to_2y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_18m_to_2y_old".to_string(), + ), + _2y_to_3y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_2y_to_3y_old".to_string(), + ), + _3y_to_4y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_3y_to_4y_old".to_string(), + ), + _4y_to_5y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_4y_to_5y_old".to_string(), + ), + _5y_to_6y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_5y_to_6y_old".to_string(), + ), + _6y_to_7y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_6y_to_7y_old".to_string(), + ), + _7y_to_8y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_7y_to_8y_old".to_string(), + ), + _8y_to_10y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_8y_to_10y_old".to_string(), + ), + _10y_to_12y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_10y_to_12y_old".to_string(), + ), + _12y_to_15y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_12y_to_15y_old".to_string(), + ), + over_15y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_15y_old".to_string(), + ), } } } @@ -8656,26 +11642,86 @@ pub struct SeriesTree_Cohorts_Utxo_UnderAge { impl SeriesTree_Cohorts_Utxo_UnderAge { pub fn new(client: Arc, base_path: String) -> Self { Self { - _1w: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_1w_old".to_string()), - _1m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_1m_old".to_string()), - _2m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_2m_old".to_string()), - _3m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_3m_old".to_string()), - _4m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_4m_old".to_string()), - _5m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_5m_old".to_string()), - _6m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_6m_old".to_string()), - _9m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_9m_old".to_string()), - _1y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_1y_old".to_string()), - _18m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_18m_old".to_string()), - _2y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_2y_old".to_string()), - _3y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_3y_old".to_string()), - _4y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_4y_old".to_string()), - _5y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_5y_old".to_string()), - _6y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_6y_old".to_string()), - _7y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_7y_old".to_string()), - _8y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_8y_old".to_string()), - _10y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_10y_old".to_string()), - _12y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_12y_old".to_string()), - _15y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_under_15y_old".to_string()), + _1w: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_1w_old".to_string(), + ), + _1m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_1m_old".to_string(), + ), + _2m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_2m_old".to_string(), + ), + _3m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_3m_old".to_string(), + ), + _4m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_4m_old".to_string(), + ), + _5m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_5m_old".to_string(), + ), + _6m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_6m_old".to_string(), + ), + _9m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_9m_old".to_string(), + ), + _1y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_1y_old".to_string(), + ), + _18m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_18m_old".to_string(), + ), + _2y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_2y_old".to_string(), + ), + _3y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_3y_old".to_string(), + ), + _4y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_4y_old".to_string(), + ), + _5y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_5y_old".to_string(), + ), + _6y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_6y_old".to_string(), + ), + _7y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_7y_old".to_string(), + ), + _8y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_8y_old".to_string(), + ), + _10y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_10y_old".to_string(), + ), + _12y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_12y_old".to_string(), + ), + _15y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_under_15y_old".to_string(), + ), } } } @@ -8707,26 +11753,86 @@ pub struct SeriesTree_Cohorts_Utxo_OverAge { impl SeriesTree_Cohorts_Utxo_OverAge { pub fn new(client: Arc, base_path: String) -> Self { Self { - _1d: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_1d_old".to_string()), - _1w: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_1w_old".to_string()), - _1m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_1m_old".to_string()), - _2m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_2m_old".to_string()), - _3m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_3m_old".to_string()), - _4m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_4m_old".to_string()), - _5m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_5m_old".to_string()), - _6m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_6m_old".to_string()), - _9m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_9m_old".to_string()), - _1y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_1y_old".to_string()), - _18m: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_18m_old".to_string()), - _2y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_2y_old".to_string()), - _3y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_3y_old".to_string()), - _4y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_4y_old".to_string()), - _5y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_5y_old".to_string()), - _6y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_6y_old".to_string()), - _7y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_7y_old".to_string()), - _8y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_8y_old".to_string()), - _10y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_10y_old".to_string()), - _12y: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_12y_old".to_string()), + _1d: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_1d_old".to_string(), + ), + _1w: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_1w_old".to_string(), + ), + _1m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_1m_old".to_string(), + ), + _2m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_2m_old".to_string(), + ), + _3m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_3m_old".to_string(), + ), + _4m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_4m_old".to_string(), + ), + _5m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_5m_old".to_string(), + ), + _6m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_6m_old".to_string(), + ), + _9m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_9m_old".to_string(), + ), + _1y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_1y_old".to_string(), + ), + _18m: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_18m_old".to_string(), + ), + _2y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_2y_old".to_string(), + ), + _3y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_3y_old".to_string(), + ), + _4y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_4y_old".to_string(), + ), + _5y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_5y_old".to_string(), + ), + _6y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_6y_old".to_string(), + ), + _7y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_7y_old".to_string(), + ), + _8y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_8y_old".to_string(), + ), + _10y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_10y_old".to_string(), + ), + _12y: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_12y_old".to_string(), + ), } } } @@ -8743,11 +11849,26 @@ pub struct SeriesTree_Cohorts_Utxo_Epoch { impl SeriesTree_Cohorts_Utxo_Epoch { pub fn new(client: Arc, base_path: String) -> Self { Self { - _0: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "epoch_0".to_string()), - _1: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "epoch_1".to_string()), - _2: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "epoch_2".to_string()), - _3: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "epoch_3".to_string()), - _4: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "epoch_4".to_string()), + _0: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "epoch_0".to_string(), + ), + _1: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "epoch_1".to_string(), + ), + _2: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "epoch_2".to_string(), + ), + _3: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "epoch_3".to_string(), + ), + _4: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "epoch_4".to_string(), + ), } } } @@ -8777,24 +11898,78 @@ pub struct SeriesTree_Cohorts_Utxo_Class { impl SeriesTree_Cohorts_Utxo_Class { pub fn new(client: Arc, base_path: String) -> Self { Self { - _2009: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2009".to_string()), - _2010: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2010".to_string()), - _2011: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2011".to_string()), - _2012: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2012".to_string()), - _2013: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2013".to_string()), - _2014: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2014".to_string()), - _2015: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2015".to_string()), - _2016: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2016".to_string()), - _2017: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2017".to_string()), - _2018: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2018".to_string()), - _2019: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2019".to_string()), - _2020: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2020".to_string()), - _2021: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2021".to_string()), - _2022: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2022".to_string()), - _2023: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2023".to_string()), - _2024: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2024".to_string()), - _2025: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2025".to_string()), - _2026: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "class_2026".to_string()), + _2009: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2009".to_string(), + ), + _2010: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2010".to_string(), + ), + _2011: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2011".to_string(), + ), + _2012: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2012".to_string(), + ), + _2013: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2013".to_string(), + ), + _2014: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2014".to_string(), + ), + _2015: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2015".to_string(), + ), + _2016: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2016".to_string(), + ), + _2017: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2017".to_string(), + ), + _2018: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2018".to_string(), + ), + _2019: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2019".to_string(), + ), + _2020: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2020".to_string(), + ), + _2021: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2021".to_string(), + ), + _2022: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2022".to_string(), + ), + _2023: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2023".to_string(), + ), + _2024: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2024".to_string(), + ), + _2025: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2025".to_string(), + ), + _2026: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "class_2026".to_string(), + ), } } } @@ -8808,8 +11983,14 @@ pub struct SeriesTree_Cohorts_Utxo_Entry { impl SeriesTree_Cohorts_Utxo_Entry { pub fn new(client: Arc, base_path: String) -> Self { Self { - discount: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "veteran".to_string()), - premium: ActivityOutputsRealizedSupplyUnrealizedPattern::new(client.clone(), "rookie".to_string()), + discount: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "veteran".to_string(), + ), + premium: ActivityOutputsRealizedSupplyUnrealizedPattern::new( + client.clone(), + "rookie".to_string(), + ), } } } @@ -8834,19 +12015,58 @@ pub struct SeriesTree_Cohorts_Utxo_OverAmount { impl SeriesTree_Cohorts_Utxo_OverAmount { pub fn new(client: Arc, base_path: String) -> Self { Self { - _1sat: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_1sat".to_string()), - _10sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_10sats".to_string()), - _100sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_100sats".to_string()), - _1k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_1k_sats".to_string()), - _10k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_10k_sats".to_string()), - _100k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_100k_sats".to_string()), - _1m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_1m_sats".to_string()), - _10m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_10m_sats".to_string()), - _1btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_1btc".to_string()), - _10btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_10btc".to_string()), - _100btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_100btc".to_string()), - _1k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_1k_btc".to_string()), - _10k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_10k_btc".to_string()), + _1sat: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_1sat".to_string(), + ), + _10sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_10sats".to_string(), + ), + _100sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_100sats".to_string(), + ), + _1k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_1k_sats".to_string(), + ), + _10k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_10k_sats".to_string(), + ), + _100k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_100k_sats".to_string(), + ), + _1m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_1m_sats".to_string(), + ), + _10m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_10m_sats".to_string(), + ), + _1btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_1btc".to_string(), + ), + _10btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_10btc".to_string(), + ), + _100btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_100btc".to_string(), + ), + _1k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_1k_btc".to_string(), + ), + _10k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_10k_btc".to_string(), + ), } } } @@ -8873,21 +12093,66 @@ pub struct SeriesTree_Cohorts_Utxo_AmountRange { impl SeriesTree_Cohorts_Utxo_AmountRange { pub fn new(client: Arc, base_path: String) -> Self { Self { - _0sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_0sats".to_string()), - _1sat_to_10sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_1sat_to_10sats".to_string()), - _10sats_to_100sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_10sats_to_100sats".to_string()), - _100sats_to_1k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_100sats_to_1k_sats".to_string()), - _1k_sats_to_10k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_1k_sats_to_10k_sats".to_string()), - _10k_sats_to_100k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_10k_sats_to_100k_sats".to_string()), - _100k_sats_to_1m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_100k_sats_to_1m_sats".to_string()), - _1m_sats_to_10m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_1m_sats_to_10m_sats".to_string()), - _10m_sats_to_1btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_10m_sats_to_1btc".to_string()), - _1btc_to_10btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_1btc_to_10btc".to_string()), - _10btc_to_100btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_10btc_to_100btc".to_string()), - _100btc_to_1k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_100btc_to_1k_btc".to_string()), - _1k_btc_to_10k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_1k_btc_to_10k_btc".to_string()), - _10k_btc_to_100k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_10k_btc_to_100k_btc".to_string()), - over_100k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_over_100k_btc".to_string()), + _0sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_0sats".to_string(), + ), + _1sat_to_10sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_1sat_to_10sats".to_string(), + ), + _10sats_to_100sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_10sats_to_100sats".to_string(), + ), + _100sats_to_1k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_100sats_to_1k_sats".to_string(), + ), + _1k_sats_to_10k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_1k_sats_to_10k_sats".to_string(), + ), + _10k_sats_to_100k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_10k_sats_to_100k_sats".to_string(), + ), + _100k_sats_to_1m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_100k_sats_to_1m_sats".to_string(), + ), + _1m_sats_to_10m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_1m_sats_to_10m_sats".to_string(), + ), + _10m_sats_to_1btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_10m_sats_to_1btc".to_string(), + ), + _1btc_to_10btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_1btc_to_10btc".to_string(), + ), + _10btc_to_100btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_10btc_to_100btc".to_string(), + ), + _100btc_to_1k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_100btc_to_1k_btc".to_string(), + ), + _1k_btc_to_10k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_1k_btc_to_10k_btc".to_string(), + ), + _10k_btc_to_100k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_10k_btc_to_100k_btc".to_string(), + ), + over_100k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_over_100k_btc".to_string(), + ), } } } @@ -8912,19 +12177,58 @@ pub struct SeriesTree_Cohorts_Utxo_UnderAmount { impl SeriesTree_Cohorts_Utxo_UnderAmount { pub fn new(client: Arc, base_path: String) -> Self { Self { - _10sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_10sats".to_string()), - _100sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_100sats".to_string()), - _1k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_1k_sats".to_string()), - _10k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_10k_sats".to_string()), - _100k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_100k_sats".to_string()), - _1m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_1m_sats".to_string()), - _10m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_10m_sats".to_string()), - _1btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_1btc".to_string()), - _10btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_10btc".to_string()), - _100btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_100btc".to_string()), - _1k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_1k_btc".to_string()), - _10k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_10k_btc".to_string()), - _100k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new(client.clone(), "utxos_under_100k_btc".to_string()), + _10sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_10sats".to_string(), + ), + _100sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_100sats".to_string(), + ), + _1k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_1k_sats".to_string(), + ), + _10k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_10k_sats".to_string(), + ), + _100k_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_100k_sats".to_string(), + ), + _1m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_1m_sats".to_string(), + ), + _10m_sats: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_10m_sats".to_string(), + ), + _1btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_1btc".to_string(), + ), + _10btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_10btc".to_string(), + ), + _100btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_100btc".to_string(), + ), + _1k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_1k_btc".to_string(), + ), + _10k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_10k_btc".to_string(), + ), + _100k_btc: ActivityOutputsRealizedSupplyUnrealizedPattern2::new( + client.clone(), + "utxos_under_100k_btc".to_string(), + ), } } } @@ -8947,17 +12251,50 @@ pub struct SeriesTree_Cohorts_Utxo_Type { impl SeriesTree_Cohorts_Utxo_Type { pub fn new(client: Arc, base_path: String) -> Self { Self { - p2pk65: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2pk65".to_string()), - p2pk33: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2pk33".to_string()), - p2pkh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2pkh".to_string()), - p2ms: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2ms".to_string()), - p2sh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2sh".to_string()), - p2wpkh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2wpkh".to_string()), - p2wsh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2wsh".to_string()), - p2tr: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2tr".to_string()), - p2a: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "p2a".to_string()), - unknown: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "unknown_outputs".to_string()), - empty: ActivityOutputsRealizedSupplyUnrealizedPattern3::new(client.clone(), "empty_outputs".to_string()), + p2pk65: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2pk65".to_string(), + ), + p2pk33: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2pk33".to_string(), + ), + p2pkh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2pkh".to_string(), + ), + p2ms: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2ms".to_string(), + ), + p2sh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2sh".to_string(), + ), + p2wpkh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2wpkh".to_string(), + ), + p2wsh: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2wsh".to_string(), + ), + p2tr: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2tr".to_string(), + ), + p2a: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "p2a".to_string(), + ), + unknown: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "unknown_outputs".to_string(), + ), + empty: ActivityOutputsRealizedSupplyUnrealizedPattern3::new( + client.clone(), + "empty_outputs".to_string(), + ), } } } @@ -8972,9 +12309,18 @@ pub struct SeriesTree_Cohorts_Utxo_Profitability { impl SeriesTree_Cohorts_Utxo_Profitability { pub fn new(client: Arc, base_path: String) -> Self { Self { - range: SeriesTree_Cohorts_Utxo_Profitability_Range::new(client.clone(), format!("{base_path}_range")), - profit: SeriesTree_Cohorts_Utxo_Profitability_Profit::new(client.clone(), format!("{base_path}_profit")), - loss: SeriesTree_Cohorts_Utxo_Profitability_Loss::new(client.clone(), format!("{base_path}_loss")), + range: SeriesTree_Cohorts_Utxo_Profitability_Range::new( + client.clone(), + format!("{base_path}_range"), + ), + profit: SeriesTree_Cohorts_Utxo_Profitability_Profit::new( + client.clone(), + format!("{base_path}_profit"), + ), + loss: SeriesTree_Cohorts_Utxo_Profitability_Loss::new( + client.clone(), + format!("{base_path}_loss"), + ), } } } @@ -9011,31 +12357,106 @@ pub struct SeriesTree_Cohorts_Utxo_Profitability_Range { impl SeriesTree_Cohorts_Utxo_Profitability_Range { pub fn new(client: Arc, base_path: String) -> Self { Self { - over_1000pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_1000pct_in_profit".to_string()), - _500pct_to_1000pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_500pct_to_1000pct_in_profit".to_string()), - _300pct_to_500pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_300pct_to_500pct_in_profit".to_string()), - _200pct_to_300pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_200pct_to_300pct_in_profit".to_string()), - _100pct_to_200pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_100pct_to_200pct_in_profit".to_string()), - _90pct_to_100pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_90pct_to_100pct_in_profit".to_string()), - _80pct_to_90pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_80pct_to_90pct_in_profit".to_string()), - _70pct_to_80pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_70pct_to_80pct_in_profit".to_string()), - _60pct_to_70pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_60pct_to_70pct_in_profit".to_string()), - _50pct_to_60pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_50pct_to_60pct_in_profit".to_string()), - _40pct_to_50pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_40pct_to_50pct_in_profit".to_string()), - _30pct_to_40pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_30pct_to_40pct_in_profit".to_string()), - _20pct_to_30pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_20pct_to_30pct_in_profit".to_string()), - _10pct_to_20pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_10pct_to_20pct_in_profit".to_string()), - _0pct_to_10pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_0pct_to_10pct_in_profit".to_string()), - _0pct_to_10pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_0pct_to_10pct_in_loss".to_string()), - _10pct_to_20pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_10pct_to_20pct_in_loss".to_string()), - _20pct_to_30pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_20pct_to_30pct_in_loss".to_string()), - _30pct_to_40pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_30pct_to_40pct_in_loss".to_string()), - _40pct_to_50pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_40pct_to_50pct_in_loss".to_string()), - _50pct_to_60pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_50pct_to_60pct_in_loss".to_string()), - _60pct_to_70pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_60pct_to_70pct_in_loss".to_string()), - _70pct_to_80pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_70pct_to_80pct_in_loss".to_string()), - _80pct_to_90pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_80pct_to_90pct_in_loss".to_string()), - _90pct_to_100pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_90pct_to_100pct_in_loss".to_string()), + over_1000pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_1000pct_in_profit".to_string(), + ), + _500pct_to_1000pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_500pct_to_1000pct_in_profit".to_string(), + ), + _300pct_to_500pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_300pct_to_500pct_in_profit".to_string(), + ), + _200pct_to_300pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_200pct_to_300pct_in_profit".to_string(), + ), + _100pct_to_200pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_100pct_to_200pct_in_profit".to_string(), + ), + _90pct_to_100pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_90pct_to_100pct_in_profit".to_string(), + ), + _80pct_to_90pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_80pct_to_90pct_in_profit".to_string(), + ), + _70pct_to_80pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_70pct_to_80pct_in_profit".to_string(), + ), + _60pct_to_70pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_60pct_to_70pct_in_profit".to_string(), + ), + _50pct_to_60pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_50pct_to_60pct_in_profit".to_string(), + ), + _40pct_to_50pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_40pct_to_50pct_in_profit".to_string(), + ), + _30pct_to_40pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_30pct_to_40pct_in_profit".to_string(), + ), + _20pct_to_30pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_20pct_to_30pct_in_profit".to_string(), + ), + _10pct_to_20pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_10pct_to_20pct_in_profit".to_string(), + ), + _0pct_to_10pct_in_profit: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_0pct_to_10pct_in_profit".to_string(), + ), + _0pct_to_10pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_0pct_to_10pct_in_loss".to_string(), + ), + _10pct_to_20pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_10pct_to_20pct_in_loss".to_string(), + ), + _20pct_to_30pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_20pct_to_30pct_in_loss".to_string(), + ), + _30pct_to_40pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_30pct_to_40pct_in_loss".to_string(), + ), + _40pct_to_50pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_40pct_to_50pct_in_loss".to_string(), + ), + _50pct_to_60pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_50pct_to_60pct_in_loss".to_string(), + ), + _60pct_to_70pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_60pct_to_70pct_in_loss".to_string(), + ), + _70pct_to_80pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_70pct_to_80pct_in_loss".to_string(), + ), + _80pct_to_90pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_80pct_to_90pct_in_loss".to_string(), + ), + _90pct_to_100pct_in_loss: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_90pct_to_100pct_in_loss".to_string(), + ), } } } @@ -9061,20 +12482,62 @@ pub struct SeriesTree_Cohorts_Utxo_Profitability_Profit { impl SeriesTree_Cohorts_Utxo_Profitability_Profit { pub fn new(client: Arc, base_path: String) -> Self { Self { - all: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_in_profit".to_string()), - _10pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_10pct_in_profit".to_string()), - _20pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_20pct_in_profit".to_string()), - _30pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_30pct_in_profit".to_string()), - _40pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_40pct_in_profit".to_string()), - _50pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_50pct_in_profit".to_string()), - _60pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_60pct_in_profit".to_string()), - _70pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_70pct_in_profit".to_string()), - _80pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_80pct_in_profit".to_string()), - _90pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_90pct_in_profit".to_string()), - _100pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_100pct_in_profit".to_string()), - _200pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_200pct_in_profit".to_string()), - _300pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_300pct_in_profit".to_string()), - _500pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_500pct_in_profit".to_string()), + all: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_in_profit".to_string(), + ), + _10pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_10pct_in_profit".to_string(), + ), + _20pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_20pct_in_profit".to_string(), + ), + _30pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_30pct_in_profit".to_string(), + ), + _40pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_40pct_in_profit".to_string(), + ), + _50pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_50pct_in_profit".to_string(), + ), + _60pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_60pct_in_profit".to_string(), + ), + _70pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_70pct_in_profit".to_string(), + ), + _80pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_80pct_in_profit".to_string(), + ), + _90pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_90pct_in_profit".to_string(), + ), + _100pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_100pct_in_profit".to_string(), + ), + _200pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_200pct_in_profit".to_string(), + ), + _300pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_300pct_in_profit".to_string(), + ), + _500pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_500pct_in_profit".to_string(), + ), } } } @@ -9095,15 +12558,42 @@ pub struct SeriesTree_Cohorts_Utxo_Profitability_Loss { impl SeriesTree_Cohorts_Utxo_Profitability_Loss { pub fn new(client: Arc, base_path: String) -> Self { Self { - all: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_in_loss".to_string()), - _10pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_10pct_in_loss".to_string()), - _20pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_20pct_in_loss".to_string()), - _30pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_30pct_in_loss".to_string()), - _40pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_40pct_in_loss".to_string()), - _50pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_50pct_in_loss".to_string()), - _60pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_60pct_in_loss".to_string()), - _70pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_70pct_in_loss".to_string()), - _80pct: NuplRealizedSupplyUnrealizedPattern::new(client.clone(), "utxos_over_80pct_in_loss".to_string()), + all: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_in_loss".to_string(), + ), + _10pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_10pct_in_loss".to_string(), + ), + _20pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_20pct_in_loss".to_string(), + ), + _30pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_30pct_in_loss".to_string(), + ), + _40pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_40pct_in_loss".to_string(), + ), + _50pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_50pct_in_loss".to_string(), + ), + _60pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_60pct_in_loss".to_string(), + ), + _70pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_70pct_in_loss".to_string(), + ), + _80pct: NuplRealizedSupplyUnrealizedPattern::new( + client.clone(), + "utxos_over_80pct_in_loss".to_string(), + ), } } } @@ -9138,29 +12628,98 @@ pub struct SeriesTree_Cohorts_Utxo_Matured { impl SeriesTree_Cohorts_Utxo_Matured { pub fn new(client: Arc, base_path: String) -> Self { Self { - under_1h: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_under_1h_old_matured_supply".to_string()), - _1h_to_1d: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_1h_to_1d_old_matured_supply".to_string()), - _1d_to_1w: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_1d_to_1w_old_matured_supply".to_string()), - _1w_to_1m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_1w_to_1m_old_matured_supply".to_string()), - _1m_to_2m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_1m_to_2m_old_matured_supply".to_string()), - _2m_to_3m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_2m_to_3m_old_matured_supply".to_string()), - _3m_to_4m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_3m_to_4m_old_matured_supply".to_string()), - _4m_to_5m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_4m_to_5m_old_matured_supply".to_string()), - _5m_to_6m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_5m_to_6m_old_matured_supply".to_string()), - _6m_to_9m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_6m_to_9m_old_matured_supply".to_string()), - _9m_to_1y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_9m_to_1y_old_matured_supply".to_string()), - _1y_to_18m: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_1y_to_18m_old_matured_supply".to_string()), - _18m_to_2y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_18m_to_2y_old_matured_supply".to_string()), - _2y_to_3y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_2y_to_3y_old_matured_supply".to_string()), - _3y_to_4y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_3y_to_4y_old_matured_supply".to_string()), - _4y_to_5y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_4y_to_5y_old_matured_supply".to_string()), - _5y_to_6y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_5y_to_6y_old_matured_supply".to_string()), - _6y_to_7y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_6y_to_7y_old_matured_supply".to_string()), - _7y_to_8y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_7y_to_8y_old_matured_supply".to_string()), - _8y_to_10y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_8y_to_10y_old_matured_supply".to_string()), - _10y_to_12y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_10y_to_12y_old_matured_supply".to_string()), - _12y_to_15y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_12y_to_15y_old_matured_supply".to_string()), - over_15y: AverageBlockCumulativeSumPattern2::new(client.clone(), "utxos_over_15y_old_matured_supply".to_string()), + under_1h: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_under_1h_old_matured_supply".to_string(), + ), + _1h_to_1d: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_1h_to_1d_old_matured_supply".to_string(), + ), + _1d_to_1w: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_1d_to_1w_old_matured_supply".to_string(), + ), + _1w_to_1m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_1w_to_1m_old_matured_supply".to_string(), + ), + _1m_to_2m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_1m_to_2m_old_matured_supply".to_string(), + ), + _2m_to_3m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_2m_to_3m_old_matured_supply".to_string(), + ), + _3m_to_4m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_3m_to_4m_old_matured_supply".to_string(), + ), + _4m_to_5m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_4m_to_5m_old_matured_supply".to_string(), + ), + _5m_to_6m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_5m_to_6m_old_matured_supply".to_string(), + ), + _6m_to_9m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_6m_to_9m_old_matured_supply".to_string(), + ), + _9m_to_1y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_9m_to_1y_old_matured_supply".to_string(), + ), + _1y_to_18m: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_1y_to_18m_old_matured_supply".to_string(), + ), + _18m_to_2y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_18m_to_2y_old_matured_supply".to_string(), + ), + _2y_to_3y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_2y_to_3y_old_matured_supply".to_string(), + ), + _3y_to_4y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_3y_to_4y_old_matured_supply".to_string(), + ), + _4y_to_5y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_4y_to_5y_old_matured_supply".to_string(), + ), + _5y_to_6y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_5y_to_6y_old_matured_supply".to_string(), + ), + _6y_to_7y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_6y_to_7y_old_matured_supply".to_string(), + ), + _7y_to_8y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_7y_to_8y_old_matured_supply".to_string(), + ), + _8y_to_10y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_8y_to_10y_old_matured_supply".to_string(), + ), + _10y_to_12y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_10y_to_12y_old_matured_supply".to_string(), + ), + _12y_to_15y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_12y_to_15y_old_matured_supply".to_string(), + ), + over_15y: AverageBlockCumulativeSumPattern2::new( + client.clone(), + "utxos_over_15y_old_matured_supply".to_string(), + ), } } } @@ -9175,9 +12734,18 @@ pub struct SeriesTree_Cohorts_Addr { impl SeriesTree_Cohorts_Addr { pub fn new(client: Arc, base_path: String) -> Self { Self { - over_amount: SeriesTree_Cohorts_Addr_OverAmount::new(client.clone(), format!("{base_path}_over_amount")), - amount_range: SeriesTree_Cohorts_Addr_AmountRange::new(client.clone(), format!("{base_path}_amount_range")), - under_amount: SeriesTree_Cohorts_Addr_UnderAmount::new(client.clone(), format!("{base_path}_under_amount")), + over_amount: SeriesTree_Cohorts_Addr_OverAmount::new( + client.clone(), + format!("{base_path}_over_amount"), + ), + amount_range: SeriesTree_Cohorts_Addr_AmountRange::new( + client.clone(), + format!("{base_path}_amount_range"), + ), + under_amount: SeriesTree_Cohorts_Addr_UnderAmount::new( + client.clone(), + format!("{base_path}_under_amount"), + ), } } } @@ -9202,19 +12770,58 @@ pub struct SeriesTree_Cohorts_Addr_OverAmount { impl SeriesTree_Cohorts_Addr_OverAmount { pub fn new(client: Arc, base_path: String) -> Self { Self { - _1sat: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_1sat".to_string()), - _10sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_10sats".to_string()), - _100sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_100sats".to_string()), - _1k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_1k_sats".to_string()), - _10k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_10k_sats".to_string()), - _100k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_100k_sats".to_string()), - _1m_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_1m_sats".to_string()), - _10m_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_10m_sats".to_string()), - _1btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_1btc".to_string()), - _10btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_10btc".to_string()), - _100btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_100btc".to_string()), - _1k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_1k_btc".to_string()), - _10k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_10k_btc".to_string()), + _1sat: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_1sat".to_string(), + ), + _10sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_10sats".to_string(), + ), + _100sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_100sats".to_string(), + ), + _1k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_1k_sats".to_string(), + ), + _10k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_10k_sats".to_string(), + ), + _100k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_100k_sats".to_string(), + ), + _1m_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_1m_sats".to_string(), + ), + _10m_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_10m_sats".to_string(), + ), + _1btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_1btc".to_string(), + ), + _10btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_10btc".to_string(), + ), + _100btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_100btc".to_string(), + ), + _1k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_1k_btc".to_string(), + ), + _10k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_10k_btc".to_string(), + ), } } } @@ -9241,21 +12848,66 @@ pub struct SeriesTree_Cohorts_Addr_AmountRange { impl SeriesTree_Cohorts_Addr_AmountRange { pub fn new(client: Arc, base_path: String) -> Self { Self { - _0sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_0sats".to_string()), - _1sat_to_10sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_1sat_to_10sats".to_string()), - _10sats_to_100sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_10sats_to_100sats".to_string()), - _100sats_to_1k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_100sats_to_1k_sats".to_string()), - _1k_sats_to_10k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_1k_sats_to_10k_sats".to_string()), - _10k_sats_to_100k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_10k_sats_to_100k_sats".to_string()), - _100k_sats_to_1m_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_100k_sats_to_1m_sats".to_string()), - _1m_sats_to_10m_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_1m_sats_to_10m_sats".to_string()), - _10m_sats_to_1btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_10m_sats_to_1btc".to_string()), - _1btc_to_10btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_1btc_to_10btc".to_string()), - _10btc_to_100btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_10btc_to_100btc".to_string()), - _100btc_to_1k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_100btc_to_1k_btc".to_string()), - _1k_btc_to_10k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_1k_btc_to_10k_btc".to_string()), - _10k_btc_to_100k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_10k_btc_to_100k_btc".to_string()), - over_100k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_over_100k_btc".to_string()), + _0sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_0sats".to_string(), + ), + _1sat_to_10sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_1sat_to_10sats".to_string(), + ), + _10sats_to_100sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_10sats_to_100sats".to_string(), + ), + _100sats_to_1k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_100sats_to_1k_sats".to_string(), + ), + _1k_sats_to_10k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_1k_sats_to_10k_sats".to_string(), + ), + _10k_sats_to_100k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_10k_sats_to_100k_sats".to_string(), + ), + _100k_sats_to_1m_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_100k_sats_to_1m_sats".to_string(), + ), + _1m_sats_to_10m_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_1m_sats_to_10m_sats".to_string(), + ), + _10m_sats_to_1btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_10m_sats_to_1btc".to_string(), + ), + _1btc_to_10btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_1btc_to_10btc".to_string(), + ), + _10btc_to_100btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_10btc_to_100btc".to_string(), + ), + _100btc_to_1k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_100btc_to_1k_btc".to_string(), + ), + _1k_btc_to_10k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_1k_btc_to_10k_btc".to_string(), + ), + _10k_btc_to_100k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_10k_btc_to_100k_btc".to_string(), + ), + over_100k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_over_100k_btc".to_string(), + ), } } } @@ -9280,19 +12932,58 @@ pub struct SeriesTree_Cohorts_Addr_UnderAmount { impl SeriesTree_Cohorts_Addr_UnderAmount { pub fn new(client: Arc, base_path: String) -> Self { Self { - _10sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_10sats".to_string()), - _100sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_100sats".to_string()), - _1k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_1k_sats".to_string()), - _10k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_10k_sats".to_string()), - _100k_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_100k_sats".to_string()), - _1m_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_1m_sats".to_string()), - _10m_sats: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_10m_sats".to_string()), - _1btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_1btc".to_string()), - _10btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_10btc".to_string()), - _100btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_100btc".to_string()), - _1k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_1k_btc".to_string()), - _10k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_10k_btc".to_string()), - _100k_btc: ActivityAddrOutputsRealizedSupplyPattern::new(client.clone(), "addrs_under_100k_btc".to_string()), + _10sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_10sats".to_string(), + ), + _100sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_100sats".to_string(), + ), + _1k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_1k_sats".to_string(), + ), + _10k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_10k_sats".to_string(), + ), + _100k_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_100k_sats".to_string(), + ), + _1m_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_1m_sats".to_string(), + ), + _10m_sats: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_10m_sats".to_string(), + ), + _1btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_1btc".to_string(), + ), + _10btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_10btc".to_string(), + ), + _100btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_100btc".to_string(), + ), + _1k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_1k_btc".to_string(), + ), + _10k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_10k_btc".to_string(), + ), + _100k_btc: ActivityAddrOutputsRealizedSupplyPattern::new( + client.clone(), + "addrs_under_100k_btc".to_string(), + ), } } } @@ -9337,20 +13028,26 @@ impl BrkClient { /// .last(10) /// .json::()?; /// ``` - pub fn series_endpoint(&self, series: impl Into, index: Index) -> SeriesEndpoint { - SeriesEndpoint::new( - self.base.clone(), - Arc::from(series.into().as_str()), - index, - ) + pub fn series_endpoint( + &self, + series: impl Into, + index: Index, + ) -> SeriesEndpoint { + SeriesEndpoint::new(self.base.clone(), Arc::from(series.into().as_str()), index) } /// Create a dynamic date-based series endpoint builder. /// /// Returns `Err` if the index is not date-based. - pub fn date_series_endpoint(&self, series: impl Into, index: Index) -> Result> { + pub fn date_series_endpoint( + &self, + series: impl Into, + index: Index, + ) -> Result> { if !index.is_date_based() { - return Err(BrkError { message: format!("{} is not a date-based index", index.name()) }); + return Err(BrkError { + message: format!("{} is not a date-based index", index.name()), + }); } Ok(DateSeriesEndpoint::new( self.base.clone(), @@ -9375,14 +13072,23 @@ impl BrkClient { } /// Fetch address hash-prefix matches from raw payload bytes matching `addr_type` length. - pub fn get_address_payload_hash_prefix_matches(&self, addr_type: OutputType, payload: &[u8], nibbles: usize) -> Result { + pub fn get_address_payload_hash_prefix_matches( + &self, + addr_type: OutputType, + payload: &[u8], + nibbles: usize, + ) -> Result { validate_address_payload_for_type(addr_type, payload)?; let prefix = address_payload_hash_prefix(payload, nibbles)?; self.get_address_hash_prefix_matches(addr_type, &prefix) } /// Fetch address hash-prefix matches for a mainnet Bitcoin address. - pub fn get_address_hash_prefix_matches_for_address(&self, address: &str, nibbles: usize) -> Result { + pub fn get_address_hash_prefix_matches_for_address( + &self, + address: &str, + nibbles: usize, + ) -> Result { let hashed = address_hash_prefix(address, nibbles)?; self.get_address_hash_prefix_matches(hashed.addr_type, &hashed.prefix) } @@ -9457,9 +13163,17 @@ impl BrkClient { /// Endpoint: `GET /api/series/list` pub fn list_series(&self, page: Option, per_page: Option) -> Result { let mut query = Vec::new(); - if let Some(v) = page { query.push(format!("page={}", v)); } - if let Some(v) = per_page { query.push(format!("per_page={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = page { + query.push(format!("page={}", v)); + } + if let Some(v) = per_page { + query.push(format!("per_page={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/series/list{}", query_str); self.base.get_json(&path) } @@ -9472,8 +13186,14 @@ impl BrkClient { pub fn search_series(&self, q: SeriesName, limit: Option) -> Result> { let mut query = Vec::new(); query.push(format!("q={}", q)); - if let Some(v) = limit { query.push(format!("limit={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = limit { + query.push(format!("limit={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/series/search{}", query_str); self.base.get_json(&path) } @@ -9492,13 +13212,33 @@ impl BrkClient { /// Fetch data for a specific series at the given index. Use query parameters to filter by date range and format (json/csv). /// /// Endpoint: `GET /api/series/{series}/{index}` - pub fn get_series(&self, series: SeriesName, index: Index, start: Option, end: Option, limit: Option, format: Option) -> Result> { + pub fn get_series( + &self, + series: SeriesName, + index: Index, + start: Option, + end: Option, + limit: Option, + format: Option, + ) -> Result> { let mut query = Vec::new(); - if let Some(v) = start { query.push(format!("start={}", v)); } - if let Some(v) = end { query.push(format!("end={}", v)); } - if let Some(v) = limit { query.push(format!("limit={}", v)); } - if let Some(v) = format { query.push(format!("format={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = start { + query.push(format!("start={}", v)); + } + if let Some(v) = end { + query.push(format!("end={}", v)); + } + if let Some(v) = limit { + query.push(format!("limit={}", v)); + } + if let Some(v) = format { + query.push(format!("format={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/series/{series}/{}{}", index.name(), query_str); if format == Some(Format::CSV) { self.base.get_text(&path).map(FormatResponse::Csv) @@ -9512,13 +13252,33 @@ impl BrkClient { /// Returns just the data array without the SeriesData wrapper. Supports the same range and format parameters as the standard endpoint. /// /// Endpoint: `GET /api/series/{series}/{index}/data` - pub fn get_series_data(&self, series: SeriesName, index: Index, start: Option, end: Option, limit: Option, format: Option) -> Result>> { + pub fn get_series_data( + &self, + series: SeriesName, + index: Index, + start: Option, + end: Option, + limit: Option, + format: Option, + ) -> Result>> { let mut query = Vec::new(); - if let Some(v) = start { query.push(format!("start={}", v)); } - if let Some(v) = end { query.push(format!("end={}", v)); } - if let Some(v) = limit { query.push(format!("limit={}", v)); } - if let Some(v) = format { query.push(format!("format={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = start { + query.push(format!("start={}", v)); + } + if let Some(v) = end { + query.push(format!("end={}", v)); + } + if let Some(v) = limit { + query.push(format!("limit={}", v)); + } + if let Some(v) = format { + query.push(format!("format={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/series/{series}/{}/data{}", index.name(), query_str); if format == Some(Format::CSV) { self.base.get_text(&path).map(FormatResponse::Csv) @@ -9533,7 +13293,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/series/{series}/{index}/latest` pub fn get_series_latest(&self, series: SeriesName, index: Index) -> Result { - self.base.get_json(&format!("/api/series/{series}/{}/latest", index.name())) + self.base + .get_json(&format!("/api/series/{series}/{}/latest", index.name())) } /// Get series data length @@ -9542,7 +13303,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/series/{series}/{index}/len` pub fn get_series_len(&self, series: SeriesName, index: Index) -> Result { - self.base.get_json(&format!("/api/series/{series}/{}/len", index.name())) + self.base + .get_json(&format!("/api/series/{series}/{}/len", index.name())) } /// Get series version @@ -9551,7 +13313,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/series/{series}/{index}/version` pub fn get_series_version(&self, series: SeriesName, index: Index) -> Result { - self.base.get_json(&format!("/api/series/{series}/{}/version", index.name())) + self.base + .get_json(&format!("/api/series/{series}/{}/version", index.name())) } /// Bulk series data @@ -9559,15 +13322,35 @@ impl BrkClient { /// Fetch multiple series in a single request. Supports filtering by index and date range. Returns an array of SeriesData objects. For a single series, use `get_series` instead. /// /// Endpoint: `GET /api/series/bulk` - pub fn get_series_bulk(&self, series: SeriesList, index: Index, start: Option, end: Option, limit: Option, format: Option) -> Result>> { + pub fn get_series_bulk( + &self, + series: SeriesList, + index: Index, + start: Option, + end: Option, + limit: Option, + format: Option, + ) -> Result>> { let mut query = Vec::new(); query.push(format!("series={}", series)); query.push(format!("index={}", index)); - if let Some(v) = start { query.push(format!("start={}", v)); } - if let Some(v) = end { query.push(format!("end={}", v)); } - if let Some(v) = limit { query.push(format!("limit={}", v)); } - if let Some(v) = format { query.push(format!("format={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = start { + query.push(format!("start={}", v)); + } + if let Some(v) = end { + query.push(format!("end={}", v)); + } + if let Some(v) = limit { + query.push(format!("limit={}", v)); + } + if let Some(v) = format { + query.push(format!("format={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/series/bulk{}", query_str); if format == Some(Format::CSV) { self.base.get_text(&path).map(FormatResponse::Csv) @@ -9603,8 +13386,14 @@ impl BrkClient { /// Endpoint: `GET /api/urpd/{cohort}` pub fn get_urpd(&self, cohort: Cohort, agg: Option) -> Result { let mut query = Vec::new(); - if let Some(v) = agg { query.push(format!("agg={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = agg { + query.push(format!("agg={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/urpd/{cohort}{}", query_str); self.base.get_json(&path) } @@ -9616,10 +13405,21 @@ impl BrkClient { /// See the URPD tag description for unit conventions and `agg` options. /// /// Endpoint: `GET /api/urpd/{cohort}/{date}` - pub fn get_urpd_at(&self, cohort: Cohort, date: &str, agg: Option) -> Result { + pub fn get_urpd_at( + &self, + cohort: Cohort, + date: &str, + agg: Option, + ) -> Result { let mut query = Vec::new(); - if let Some(v) = agg { query.push(format!("agg={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = agg { + query.push(format!("agg={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/urpd/{cohort}/{date}{}", query_str); self.base.get_json(&path) } @@ -9632,7 +13432,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/difficulty-adjustment` pub fn get_difficulty_adjustment(&self) -> Result { - self.base.get_json(&format!("/api/v1/difficulty-adjustment")) + self.base + .get_json(&format!("/api/v1/difficulty-adjustment")) } /// Current BTC price @@ -9655,8 +13456,14 @@ impl BrkClient { /// Endpoint: `GET /api/v1/historical-price` pub fn get_historical_price(&self, timestamp: Option) -> Result { let mut query = Vec::new(); - if let Some(v) = timestamp { query.push(format!("timestamp={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + if let Some(v) = timestamp { + query.push(format!("timestamp={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/v1/historical-price{}", query_str); self.base.get_json(&path) } @@ -9666,9 +13473,14 @@ impl BrkClient { /// Find addresses by address type and by the first 1-16 hex nibbles of RapidHash v3 over the raw address payload bytes. Intended for privacy-preserving client-side wallet discovery without sending raw addresses or xpubs. Fetch metadata for the returned addresses through `/api/address/{address}`. /// /// Endpoint: `GET /api/address/hash-prefix/{addr_type}/{prefix}` - pub fn get_address_hash_prefix_matches(&self, addr_type: OutputType, prefix: &str) -> Result { + pub fn get_address_hash_prefix_matches( + &self, + addr_type: OutputType, + prefix: &str, + ) -> Result { let addr_type = address_payload_type_path(addr_type)?; - self.base.get_json(&format!("/api/address/hash-prefix/{addr_type}/{prefix}")) + self.base + .get_json(&format!("/api/address/hash-prefix/{addr_type}/{prefix}")) } /// Address information @@ -9701,7 +13513,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/address/{address}/txs/chain` pub fn get_address_confirmed_txs(&self, address: Addr) -> Result> { - self.base.get_json(&format!("/api/address/{address}/txs/chain")) + self.base + .get_json(&format!("/api/address/{address}/txs/chain")) } /// Address confirmed transactions (paginated) @@ -9711,8 +13524,13 @@ impl BrkClient { /// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-address-transactions-chain)* /// /// Endpoint: `GET /api/address/{address}/txs/chain/{after_txid}` - pub fn get_address_confirmed_txs_after(&self, address: Addr, after_txid: Txid) -> Result> { - self.base.get_json(&format!("/api/address/{address}/txs/chain/{after_txid}")) + pub fn get_address_confirmed_txs_after( + &self, + address: Addr, + after_txid: Txid, + ) -> Result> { + self.base + .get_json(&format!("/api/address/{address}/txs/chain/{after_txid}")) } /// Address mempool transactions @@ -9723,7 +13541,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/address/{address}/txs/mempool` pub fn get_address_mempool_txs(&self, address: Addr) -> Result> { - self.base.get_json(&format!("/api/address/{address}/txs/mempool")) + self.base + .get_json(&format!("/api/address/{address}/txs/mempool")) } /// Address UTXOs @@ -9745,7 +13564,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/validate-address/{address}` pub fn validate_address(&self, address: &str) -> Result { - self.base.get_json(&format!("/api/v1/validate-address/{address}")) + self.base + .get_json(&format!("/api/v1/validate-address/{address}")) } /// Block information @@ -9800,7 +13620,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/blocks/timestamp/{timestamp}` pub fn get_block_by_timestamp(&self, timestamp: Timestamp) -> Result { - self.base.get_json(&format!("/api/v1/mining/blocks/timestamp/{timestamp}")) + self.base + .get_json(&format!("/api/v1/mining/blocks/timestamp/{timestamp}")) } /// Raw block @@ -9855,7 +13676,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/block/{hash}/txid/{index}` pub fn get_block_txid(&self, hash: BlockHash, index: BlockTxIndex) -> Result { - self.base.get_text(&format!("/api/block/{hash}/txid/{index}")) + self.base + .get_text(&format!("/api/block/{hash}/txid/{index}")) } /// Block transaction IDs @@ -9887,8 +13709,13 @@ impl BrkClient { /// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-block-transactions)* /// /// Endpoint: `GET /api/block/{hash}/txs/{start_index}` - pub fn get_block_txs_from_index(&self, hash: BlockHash, start_index: BlockTxIndex) -> Result> { - self.base.get_json(&format!("/api/block/{hash}/txs/{start_index}")) + pub fn get_block_txs_from_index( + &self, + hash: BlockHash, + start_index: BlockTxIndex, + ) -> Result> { + self.base + .get_json(&format!("/api/block/{hash}/txs/{start_index}")) } /// Recent blocks @@ -9954,7 +13781,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/pools/{time_period}` pub fn get_pool_stats(&self, time_period: TimePeriod) -> Result { - self.base.get_json(&format!("/api/v1/mining/pools/{time_period}")) + self.base + .get_json(&format!("/api/v1/mining/pools/{time_period}")) } /// Mining pool details @@ -9976,7 +13804,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/hashrate/pools` pub fn get_pools_hashrate(&self) -> Result> { - self.base.get_json(&format!("/api/v1/mining/hashrate/pools")) + self.base + .get_json(&format!("/api/v1/mining/hashrate/pools")) } /// All pools hashrate @@ -9986,8 +13815,12 @@ impl BrkClient { /// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-mining-pool-hashrates)* /// /// Endpoint: `GET /api/v1/mining/hashrate/pools/{time_period}` - pub fn get_pools_hashrate_by_period(&self, time_period: TimePeriod) -> Result> { - self.base.get_json(&format!("/api/v1/mining/hashrate/pools/{time_period}")) + pub fn get_pools_hashrate_by_period( + &self, + time_period: TimePeriod, + ) -> Result> { + self.base + .get_json(&format!("/api/v1/mining/hashrate/pools/{time_period}")) } /// Mining pool hashrate @@ -9998,7 +13831,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/pool/{slug}/hashrate` pub fn get_pool_hashrate(&self, slug: PoolSlug) -> Result> { - self.base.get_json(&format!("/api/v1/mining/pool/{slug}/hashrate")) + self.base + .get_json(&format!("/api/v1/mining/pool/{slug}/hashrate")) } /// Mining pool blocks @@ -10009,7 +13843,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/pool/{slug}/blocks` pub fn get_pool_blocks(&self, slug: PoolSlug) -> Result> { - self.base.get_json(&format!("/api/v1/mining/pool/{slug}/blocks")) + self.base + .get_json(&format!("/api/v1/mining/pool/{slug}/blocks")) } /// Mining pool blocks from height @@ -10020,7 +13855,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/pool/{slug}/blocks/{height}` pub fn get_pool_blocks_from(&self, slug: PoolSlug, height: Height) -> Result> { - self.base.get_json(&format!("/api/v1/mining/pool/{slug}/blocks/{height}")) + self.base + .get_json(&format!("/api/v1/mining/pool/{slug}/blocks/{height}")) } /// Network hashrate (all time) @@ -10042,7 +13878,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/hashrate/{time_period}` pub fn get_hashrate_by_period(&self, time_period: TimePeriod) -> Result { - self.base.get_json(&format!("/api/v1/mining/hashrate/{time_period}")) + self.base + .get_json(&format!("/api/v1/mining/hashrate/{time_period}")) } /// Difficulty adjustments (all time) @@ -10053,7 +13890,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/difficulty-adjustments` pub fn get_difficulty_adjustments(&self) -> Result> { - self.base.get_json(&format!("/api/v1/mining/difficulty-adjustments")) + self.base + .get_json(&format!("/api/v1/mining/difficulty-adjustments")) } /// Difficulty adjustments @@ -10063,8 +13901,13 @@ impl BrkClient { /// *[Mempool.space docs](https://mempool.space/docs/api/rest#get-difficulty-adjustments)* /// /// Endpoint: `GET /api/v1/mining/difficulty-adjustments/{time_period}` - pub fn get_difficulty_adjustments_by_period(&self, time_period: TimePeriod) -> Result> { - self.base.get_json(&format!("/api/v1/mining/difficulty-adjustments/{time_period}")) + pub fn get_difficulty_adjustments_by_period( + &self, + time_period: TimePeriod, + ) -> Result> { + self.base.get_json(&format!( + "/api/v1/mining/difficulty-adjustments/{time_period}" + )) } /// Mining reward statistics @@ -10075,7 +13918,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/reward-stats/{block_count}` pub fn get_reward_stats(&self, block_count: i64) -> Result { - self.base.get_json(&format!("/api/v1/mining/reward-stats/{block_count}")) + self.base + .get_json(&format!("/api/v1/mining/reward-stats/{block_count}")) } /// Block fees @@ -10086,7 +13930,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/blocks/fees/{time_period}` pub fn get_block_fees(&self, time_period: TimePeriod) -> Result> { - self.base.get_json(&format!("/api/v1/mining/blocks/fees/{time_period}")) + self.base + .get_json(&format!("/api/v1/mining/blocks/fees/{time_period}")) } /// Block rewards @@ -10097,7 +13942,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/blocks/rewards/{time_period}` pub fn get_block_rewards(&self, time_period: TimePeriod) -> Result> { - self.base.get_json(&format!("/api/v1/mining/blocks/rewards/{time_period}")) + self.base + .get_json(&format!("/api/v1/mining/blocks/rewards/{time_period}")) } /// Block fee rates @@ -10108,7 +13954,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/blocks/fee-rates/{time_period}` pub fn get_block_fee_rates(&self, time_period: TimePeriod) -> Result> { - self.base.get_json(&format!("/api/v1/mining/blocks/fee-rates/{time_period}")) + self.base + .get_json(&format!("/api/v1/mining/blocks/fee-rates/{time_period}")) } /// Block sizes and weights @@ -10119,7 +13966,9 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mining/blocks/sizes-weights/{time_period}` pub fn get_block_sizes_weights(&self, time_period: TimePeriod) -> Result { - self.base.get_json(&format!("/api/v1/mining/blocks/sizes-weights/{time_period}")) + self.base.get_json(&format!( + "/api/v1/mining/blocks/sizes-weights/{time_period}" + )) } /// Projected mempool blocks @@ -10225,7 +14074,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mempool/block-template` pub fn get_block_template(&self) -> Result { - self.base.get_json(&format!("/api/v1/mempool/block-template")) + self.base + .get_json(&format!("/api/v1/mempool/block-template")) } /// Block template diff since hash @@ -10234,7 +14084,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/v1/mempool/block-template/diff/{hash}` pub fn get_block_template_diff(&self, hash: NextBlockHash) -> Result { - self.base.get_json(&format!("/api/v1/mempool/block-template/diff/{hash}")) + self.base + .get_json(&format!("/api/v1/mempool/block-template/diff/{hash}")) } /// Live BTC/USD price @@ -10261,7 +14112,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/oracle/histogram/payments/live` pub fn get_oracle_histogram_payments_live(&self) -> Result> { - self.base.get_json(&format!("/api/oracle/histogram/payments/live")) + self.base + .get_json(&format!("/api/oracle/histogram/payments/live")) } /// Payment output histogram at height or day @@ -10270,7 +14122,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/oracle/histogram/payments/{point}` pub fn get_oracle_histogram_payments(&self, point: &str) -> Result> { - self.base.get_json(&format!("/api/oracle/histogram/payments/{point}")) + self.base + .get_json(&format!("/api/oracle/histogram/payments/{point}")) } /// Live output value histogram @@ -10279,7 +14132,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/oracle/histogram/outputs/live` pub fn get_oracle_histogram_outputs_live(&self) -> Result> { - self.base.get_json(&format!("/api/oracle/histogram/outputs/live")) + self.base + .get_json(&format!("/api/oracle/histogram/outputs/live")) } /// Output value histogram at height or day @@ -10288,7 +14142,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/oracle/histogram/outputs/{point}` pub fn get_oracle_histogram_outputs(&self, point: &str) -> Result> { - self.base.get_json(&format!("/api/oracle/histogram/outputs/{point}")) + self.base + .get_json(&format!("/api/oracle/histogram/outputs/{point}")) } /// Txid by index @@ -10352,7 +14207,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/tx/{txid}/merkleblock-proof` pub fn get_tx_merkleblock_proof(&self, txid: Txid) -> Result { - self.base.get_text(&format!("/api/tx/{txid}/merkleblock-proof")) + self.base + .get_text(&format!("/api/tx/{txid}/merkleblock-proof")) } /// Transaction merkle proof @@ -10374,7 +14230,8 @@ impl BrkClient { /// /// Endpoint: `GET /api/tx/{txid}/outspend/{vout}` pub fn get_tx_outspend(&self, txid: Txid, vout: Vout) -> Result { - self.base.get_json(&format!("/api/tx/{txid}/outspend/{vout}")) + self.base + .get_json(&format!("/api/tx/{txid}/outspend/{vout}")) } /// All output spend statuses @@ -10419,8 +14276,14 @@ impl BrkClient { /// Endpoint: `GET /api/v1/transaction-times` pub fn get_transaction_times(&self, txId: &[Txid]) -> Result> { let mut query = Vec::new(); - for v in txId { query.push(format!("txId[]={}", v)); } - let query_str = if query.is_empty() { String::new() } else { format!("?{}", query.join("&")) }; + for v in txId { + query.push(format!("txId[]={}", v)); + } + let query_str = if query.is_empty() { + String::new() + } else { + format!("?{}", query.join("&")) + }; let path = format!("/api/v1/transaction-times{}", query_str); self.base.get_json(&path) } @@ -10453,5 +14316,4 @@ impl BrkClient { pub fn get_api(&self) -> Result { self.base.get_json(&format!("/api.json")) } - } diff --git a/crates/brk_computer/src/distribution/addr/indexes/any.rs b/crates/brk_computer/src/distribution/addr/indexes/any.rs index ed5ad2e31..9d4fd12b0 100644 --- a/crates/brk_computer/src/distribution/addr/indexes/any.rs +++ b/crates/brk_computer/src/distribution/addr/indexes/any.rs @@ -11,8 +11,8 @@ use brk_types::{ use rayon::prelude::*; use rustc_hash::FxHashMap; use vecdb::{ - AnyStoredVec, AnyVec, BytesVec, Database, ImportOptions, ImportableVec, ReadableVec, Reader, - Rw, Stamp, StorageMode, WritableVec, + AnyStoredVec, AnyVec, BytesVec, Database, ImportOptions, ImportableVec, ReadableVec, Rw, Stamp, + StorageMode, WritableVec, }; use super::super::AddrTypeToTypeIndexMap; @@ -57,15 +57,6 @@ macro_rules! define_any_addr_indexes_vecs { Ok(()) } - /// Get address index for a given type and type_index. - /// Uses get_any_or_read_at to check updated layer (needed after rollback). - pub(crate) fn get(&self, addr_type: OutputType, type_index: TypeIndex, reader: &Reader) -> Result { - match addr_type { - $(OutputType::$variant => Ok(self.$field.get_any_or_read_at(type_index.into(), reader)?.unwrap()),)* - _ => unreachable!("Invalid addr type: {:?}", addr_type), - } - } - /// Returns a parallel iterator over all vecs for parallel writing. pub(crate) fn par_iter_mut(&mut self) -> impl ParallelIterator { vec![$(&mut self.$field as &mut dyn AnyStoredVec),*].into_par_iter() diff --git a/crates/brk_computer/src/distribution/block/cache/addr.rs b/crates/brk_computer/src/distribution/block/cache/addr.rs index 69c448668..605aa767c 100644 --- a/crates/brk_computer/src/distribution/block/cache/addr.rs +++ b/crates/brk_computer/src/distribution/block/cache/addr.rs @@ -1,5 +1,4 @@ use brk_cohort::ByAddrType; -use brk_error::Result; use brk_types::{ AnyAddrDataIndexEnum, EmptyAddrData, FundedAddrData, OutputType, TxIndex, TypeIndex, }; @@ -99,38 +98,29 @@ pub(crate) fn load_uncached_addr_data( vr: &VecsReaders, any_addr_indexes: &AnyAddrIndexesVecs, addrs_data: &AddrsDataVecs, -) -> Result>> { +) -> Option> { // Check if this is a new address (type_index >= first for this height) let first = *first_addr_indexes.get(addr_type).unwrap(); if first <= type_index { - return Ok(Some(WithAddrDataSource::New(FundedAddrData::default()))); + return Some(WithAddrDataSource::New(FundedAddrData::default())); } // Skip if already in cache if cache.contains(addr_type, type_index) { - return Ok(None); + return None; } // Read from storage - let reader = vr.addr_reader(addr_type); - let any_addr_index = any_addr_indexes.get(addr_type, type_index, reader)?; + let any_addr_index = vr.any_addr_index(any_addr_indexes, addr_type, type_index); - Ok(Some(match any_addr_index.to_enum() { + Some(match any_addr_index.to_enum() { AnyAddrDataIndexEnum::Funded(funded_index) => { - let reader = &vr.any_addr_index_to_any_addr_data.funded; - let funded_data = addrs_data - .funded - .get_any_or_read_at(funded_index.into(), reader)? - .unwrap(); + let funded_data = vr.funded_data(addrs_data, funded_index); WithAddrDataSource::FromFunded(funded_index, funded_data) } AnyAddrDataIndexEnum::Empty(empty_index) => { - let reader = &vr.any_addr_index_to_any_addr_data.empty; - let empty_data = addrs_data - .empty - .get_any_or_read_at(empty_index.into(), reader)? - .unwrap(); + let empty_data = vr.empty_data(addrs_data, empty_index); WithAddrDataSource::FromEmpty(empty_index, empty_data.into()) } - })) + }) } diff --git a/crates/brk_computer/src/distribution/block/utxo/inputs.rs b/crates/brk_computer/src/distribution/block/utxo/inputs.rs index 7ea356404..2f0243dc9 100644 --- a/crates/brk_computer/src/distribution/block/utxo/inputs.rs +++ b/crates/brk_computer/src/distribution/block/utxo/inputs.rs @@ -1,5 +1,4 @@ use brk_cohort::ByAddrType; -use brk_error::Result; use brk_types::{FundedAddrData, Height, OutputType, Sats, TxIndex, TypeIndex}; use rayon::prelude::*; use rustc_hash::FxHashMap; @@ -56,8 +55,8 @@ pub(crate) fn process_inputs( vr: &VecsReaders, any_addr_indexes: &AnyAddrIndexesVecs, addrs_data: &AddrsDataVecs, -) -> Result { - let map_fn = |local_idx: usize| -> Result<_> { +) -> InputsResult { + let map_fn = |local_idx: usize| { let tx_index = txin_index_to_tx_index[local_idx]; let prev_height = txin_index_to_prev_height[local_idx]; @@ -65,7 +64,7 @@ pub(crate) fn process_inputs( let input_type = txin_index_to_output_type[local_idx]; if input_type.is_not_addr() { - return Ok((prev_height, value, input_type, None)); + return (prev_height, value, input_type, None); } let type_index = txin_index_to_type_index[local_idx]; @@ -79,23 +78,20 @@ pub(crate) fn process_inputs( vr, any_addr_indexes, addrs_data, - )?; + ); - Ok(( + ( prev_height, value, input_type, Some((type_index, tx_index, value, addr_data_opt)), - )) + ) }; let items: Vec<_> = if input_count < 128 { - (0..input_count).map(map_fn).collect::>>()? + (0..input_count).map(map_fn).collect() } else { - (0..input_count) - .into_par_iter() - .map(map_fn) - .collect::>>()? + (0..input_count).into_par_iter().map(map_fn).collect() }; // Phase 2: Sequential accumulation - no merge overhead @@ -140,10 +136,10 @@ pub(crate) fn process_inputs( } } - Ok(InputsResult { + InputsResult { height_to_sent, sent_data, addr_data, tx_index_vecs, - }) + } } diff --git a/crates/brk_computer/src/distribution/block/utxo/outputs.rs b/crates/brk_computer/src/distribution/block/utxo/outputs.rs index 94245c030..600d638ec 100644 --- a/crates/brk_computer/src/distribution/block/utxo/outputs.rs +++ b/crates/brk_computer/src/distribution/block/utxo/outputs.rs @@ -1,5 +1,4 @@ use brk_cohort::ByAddrType; -use brk_error::Result; use brk_types::{FundedAddrData, Sats, TxIndex, TypeIndex}; use rayon::prelude::*; use smallvec::SmallVec; @@ -43,17 +42,17 @@ pub(crate) fn process_outputs( vr: &VecsReaders, any_addr_indexes: &AnyAddrIndexesVecs, addrs_data: &AddrsDataVecs, -) -> Result { +) -> OutputsResult { let output_count = txout_data_vec.len(); // Phase 1: Addr lookups (mmap reads) — parallel for large blocks, sequential for small - let map_fn = |local_idx: usize| -> Result<_> { + let map_fn = |local_idx: usize| { let txout_data = &txout_data_vec[local_idx]; let value = txout_data.value; let output_type = txout_data.output_type; if output_type.is_not_addr() { - return Ok((value, output_type, None)); + return (value, output_type, None); } let type_index = txout_data.type_index; @@ -67,22 +66,19 @@ pub(crate) fn process_outputs( vr, any_addr_indexes, addrs_data, - )?; + ); - Ok(( + ( value, output_type, Some((type_index, tx_index, value, addr_data_opt)), - )) + ) }; let items: Vec<_> = if output_count < 128 { - (0..output_count).map(map_fn).collect::>>()? + (0..output_count).map(map_fn).collect() } else { - (0..output_count) - .into_par_iter() - .map(map_fn) - .collect::>>()? + (0..output_count).into_par_iter().map(map_fn).collect() }; // Phase 2: Sequential accumulation @@ -117,10 +113,10 @@ pub(crate) fn process_outputs( } } - Ok(OutputsResult { + OutputsResult { transacted, received_data, addr_data, tx_index_vecs, - }) + } } diff --git a/crates/brk_computer/src/distribution/compute/block_loop.rs b/crates/brk_computer/src/distribution/compute/block_loop.rs index fb818997b..4f60ccaba 100644 --- a/crates/brk_computer/src/distribution/compute/block_loop.rs +++ b/crates/brk_computer/src/distribution/compute/block_loop.rs @@ -266,13 +266,13 @@ pub(crate) fn process_blocks( // Collection (build tx_index mappings + bulk mmap reads) is merged into the // processing closures so outputs and inputs collection overlap each other // and tick-tock, instead of running sequentially before the join. - let (matured, oi_result) = rayon::join( + let (matured, (outputs_result, inputs_result)) = rayon::join( || { vecs.utxo_cohorts .tick_tock_next_block(chain_state, timestamp) }, - || -> Result<_> { - let (outputs_result, inputs_result) = rayon::join( + || { + rayon::join( || { let txout_index_to_tx_index = txout_to_tx_index_buf.build( first_tx_index, @@ -291,7 +291,7 @@ pub(crate) fn process_blocks( &vecs.addrs_data, ) }, - || -> Result<_> { + || { if input_count > 1 { let txin_index_to_tx_index = txin_to_tx_index_buf.build( first_tx_index, @@ -322,19 +322,17 @@ pub(crate) fn process_blocks( &vecs.addrs_data, ) } else { - Ok(InputsResult { + InputsResult { height_to_sent: Default::default(), sent_data: Default::default(), addr_data: Default::default(), tx_index_vecs: Default::default(), - }) + } } }, - ); - Ok((outputs_result?, inputs_result?)) + ) }, ); - let (outputs_result, inputs_result) = oi_result?; // Merge new address data into current cache cache.merge_funded(outputs_result.addr_data); diff --git a/crates/brk_computer/src/distribution/compute/readers.rs b/crates/brk_computer/src/distribution/compute/readers.rs index 563b40085..3ee467dd0 100644 --- a/crates/brk_computer/src/distribution/compute/readers.rs +++ b/crates/brk_computer/src/distribution/compute/readers.rs @@ -1,7 +1,10 @@ -use brk_cohort::{ByAddrType, ByAnyAddr}; use brk_indexer::Indexer; -use brk_types::{Height, OutPoint, OutputType, Sats, StoredU64, TxIndex, TypeIndex}; -use vecdb::{ReadableVec, Reader, VecIndex}; +use brk_types::{ + AnyAddrIndex, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, Height, OutPoint, + OutputType, P2AAddrIndex, P2PK33AddrIndex, P2PK65AddrIndex, P2PKHAddrIndex, P2SHAddrIndex, + P2TRAddrIndex, P2WPKHAddrIndex, P2WSHAddrIndex, Sats, StoredU64, TxIndex, TypeIndex, +}; +use vecdb::{BytesVecReader, ReadableVec, VecIndex}; use crate::{ distribution::{ @@ -159,35 +162,67 @@ impl<'a> TxInReaders<'a> { /// Cached readers for stateful vectors. pub struct VecsReaders { - pub addr_type_index_to_any_addr_index: ByAddrType, - pub any_addr_index_to_any_addr_data: ByAnyAddr, + p2a: BytesVecReader, + p2pk33: BytesVecReader, + p2pk65: BytesVecReader, + p2pkh: BytesVecReader, + p2sh: BytesVecReader, + p2tr: BytesVecReader, + p2wpkh: BytesVecReader, + p2wsh: BytesVecReader, + funded: BytesVecReader, + empty: BytesVecReader, } impl VecsReaders { pub(crate) fn new(any_addr_indexes: &AnyAddrIndexesVecs, addrs_data: &AddrsDataVecs) -> Self { Self { - addr_type_index_to_any_addr_index: ByAddrType { - p2a: any_addr_indexes.p2a.create_reader(), - p2pk33: any_addr_indexes.p2pk33.create_reader(), - p2pk65: any_addr_indexes.p2pk65.create_reader(), - p2pkh: any_addr_indexes.p2pkh.create_reader(), - p2sh: any_addr_indexes.p2sh.create_reader(), - p2tr: any_addr_indexes.p2tr.create_reader(), - p2wpkh: any_addr_indexes.p2wpkh.create_reader(), - p2wsh: any_addr_indexes.p2wsh.create_reader(), - }, - any_addr_index_to_any_addr_data: ByAnyAddr { - funded: addrs_data.funded.create_reader(), - empty: addrs_data.empty.create_reader(), - }, + p2a: any_addr_indexes.p2a.reader(), + p2pk33: any_addr_indexes.p2pk33.reader(), + p2pk65: any_addr_indexes.p2pk65.reader(), + p2pkh: any_addr_indexes.p2pkh.reader(), + p2sh: any_addr_indexes.p2sh.reader(), + p2tr: any_addr_indexes.p2tr.reader(), + p2wpkh: any_addr_indexes.p2wpkh.reader(), + p2wsh: any_addr_indexes.p2wsh.reader(), + funded: addrs_data.funded.reader(), + empty: addrs_data.empty.reader(), } } - /// Get reader for specific address type. - pub(crate) fn addr_reader(&self, addr_type: OutputType) -> &Reader { - self.addr_type_index_to_any_addr_index - .get(addr_type) - .unwrap() + /// Read the unified address index, including uncommitted updates after rollback. + pub(crate) fn any_addr_index( + &self, + vecs: &AnyAddrIndexesVecs, + addr_type: OutputType, + type_index: TypeIndex, + ) -> AnyAddrIndex { + let index = match addr_type { + OutputType::P2A => vecs.p2a.get_with_reader(type_index.into(), &self.p2a), + OutputType::P2PK33 => vecs.p2pk33.get_with_reader(type_index.into(), &self.p2pk33), + OutputType::P2PK65 => vecs.p2pk65.get_with_reader(type_index.into(), &self.p2pk65), + OutputType::P2PKH => vecs.p2pkh.get_with_reader(type_index.into(), &self.p2pkh), + OutputType::P2SH => vecs.p2sh.get_with_reader(type_index.into(), &self.p2sh), + OutputType::P2TR => vecs.p2tr.get_with_reader(type_index.into(), &self.p2tr), + OutputType::P2WPKH => vecs.p2wpkh.get_with_reader(type_index.into(), &self.p2wpkh), + OutputType::P2WSH => vecs.p2wsh.get_with_reader(type_index.into(), &self.p2wsh), + _ => unreachable!("invalid address type: {addr_type:?}"), + }; + index.unwrap() + } + + #[inline] + pub(crate) fn funded_data( + &self, + vecs: &AddrsDataVecs, + index: FundedAddrIndex, + ) -> FundedAddrData { + vecs.funded.get_with_reader(index, &self.funded).unwrap() + } + + #[inline] + pub(crate) fn empty_data(&self, vecs: &AddrsDataVecs, index: EmptyAddrIndex) -> EmptyAddrData { + vecs.empty.get_with_reader(index, &self.empty).unwrap() } } diff --git a/crates/brk_computer/src/distribution/vecs.rs b/crates/brk_computer/src/distribution/vecs.rs index f8b534a73..61f5a88ef 100644 --- a/crates/brk_computer/src/distribution/vecs.rs +++ b/crates/brk_computer/src/distribution/vecs.rs @@ -350,6 +350,44 @@ impl Vecs { ) -> Result<()> { self.db.sync_bg_tasks()?; + let base_version = VERSION + + [ + prices.spot.cents.height.version(), + indexes.timestamp.monotonic.version(), + indexer.vecs.transactions.first_tx_index.version(), + indexer.vecs.outputs.first_txout_index.version(), + indexer.vecs.inputs.first_txin_index.version(), + transactions.count.total.block.version(), + outputs.count.total.sum.version(), + inputs.count.sum.version(), + indexes.tx_index.output_count.version(), + indexes.tx_index.input_count.version(), + indexer.vecs.outputs.value.version(), + indexer.vecs.outputs.output_type.version(), + indexer.vecs.outputs.type_index.version(), + inputs.spent.value.version(), + indexer.vecs.inputs.outpoint.version(), + indexer.vecs.inputs.output_type.version(), + indexer.vecs.inputs.type_index.version(), + indexer.vecs.addrs.p2pk65.first_index.version(), + indexer.vecs.addrs.p2pk33.first_index.version(), + indexer.vecs.addrs.p2pkh.first_index.version(), + indexer.vecs.addrs.p2sh.first_index.version(), + indexer.vecs.addrs.p2wpkh.first_index.version(), + indexer.vecs.addrs.p2wsh.first_index.version(), + indexer.vecs.addrs.p2tr.first_index.version(), + indexer.vecs.addrs.p2a.first_index.version(), + ] + .into_iter() + .sum::(); + + debug!("validating computed versions"); + self.supply_state + .validate_computed_version_or_reset(base_version)?; + self.utxo_cohorts.validate_computed_versions(base_version)?; + self.addr_cohorts.validate_computed_versions(base_version)?; + debug!("computed versions validated"); + let starting_lengths = indexer.safe_lengths(); // 1. Find minimum height we have data for across stateful vecs @@ -503,13 +541,6 @@ impl Vecs { recovered_height }; - // 2c. Validate computed versions - debug!("validating computed versions"); - let base_version = VERSION; - self.utxo_cohorts.validate_computed_versions(base_version)?; - self.addr_cohorts.validate_computed_versions(base_version)?; - debug!("computed versions validated"); - // 3. Get last height from indexer let last_height = Height::from(indexer.vecs.blocks.blockhash.len().saturating_sub(1)); debug!( diff --git a/crates/brk_computer/src/inputs/spent/compute.rs b/crates/brk_computer/src/inputs/spent/compute.rs index ad427703b..32d566884 100644 --- a/crates/brk_computer/src/inputs/spent/compute.rs +++ b/crates/brk_computer/src/inputs/spent/compute.rs @@ -70,8 +70,7 @@ impl Vecs { if entry.tx_index.is_coinbase() { break; } - entry.txout_index = - first_txout_index_reader.get(entry.tx_index.to_usize()) + entry.vout; + entry.txout_index = first_txout_index_reader.get(entry.tx_index) + entry.vout; } // Sort 2: by txout_index (sequential value reads) @@ -80,7 +79,7 @@ impl Vecs { if entry.txout_index.is_coinbase() { break; } - entry.value = value_reader.get(entry.txout_index.to_usize()); + entry.value = value_reader.get(entry.txout_index); } // Scatter-write to output buffers using original_idx (avoids Sort 3) diff --git a/crates/brk_computer/src/investing/import.rs b/crates/brk_computer/src/investing/import.rs index b9cb69e31..083eb80aa 100644 --- a/crates/brk_computer/src/investing/import.rs +++ b/crates/brk_computer/src/investing/import.rs @@ -5,7 +5,7 @@ use brk_types::{ Bitcoin, Cents, Date, Day1, Dollars, Height, PartsPerMillionSigned64, Sats, Version, }; use vecdb::{ - BinaryTransform, CachedBoxedVec, CheckedSub, EagerVec, ImportableVec, PcoVec, + AnyVec, BinaryTransform, CachedBoxedVec, CheckedSub, EagerVec, ImportableVec, PcoVec, ReadableCloneableVec, ReadableVec, TypedVec, VecIndex, }; @@ -33,8 +33,9 @@ impl Vecs { let db = open_db(parent_path, super::DB_NAME, 50_000)?; let version = parent_version; + let sats_cumulative_version = version + prices.split.close.usd.day1.version(); let sats_cumulative: EagerVec> = - ImportableVec::forced_import(&db, "dca_sats_cumulative", version)?; + ImportableVec::forced_import(&db, "dca_sats_cumulative", sats_cumulative_version)?; let sats_per_day = LazyPreviousDeltaVec::new( "dca_sats_per_day", version, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index c90272a01..e76268407 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -121,6 +121,7 @@ impl Computer { let mining = Box::new(mining::Vecs::forced_import( &computed_path, VERSION, + indexer, &indexes, &cached_starts, )?); diff --git a/crates/brk_computer/src/mining/import.rs b/crates/brk_computer/src/mining/import.rs index b1adf3d76..8ee608ccf 100644 --- a/crates/brk_computer/src/mining/import.rs +++ b/crates/brk_computer/src/mining/import.rs @@ -1,6 +1,7 @@ use std::path::Path; use brk_error::Result; +use brk_indexer::Indexer; use brk_types::Version; use crate::{ @@ -17,13 +18,14 @@ impl Vecs { pub(crate) fn forced_import( parent_path: &Path, parent_version: Version, + indexer: &Indexer, indexes: &indexes::Vecs, cached_starts: &Windows<&WindowStartVec>, ) -> Result { let db = open_db(parent_path, super::DB_NAME, 1_000_000)?; let version = parent_version; - let rewards = RewardsVecs::forced_import(&db, version, indexes, cached_starts)?; + let rewards = RewardsVecs::forced_import(&db, version, indexer, indexes, cached_starts)?; let hashrate = HashrateVecs::forced_import(&db, version, indexes)?; let this = Self { diff --git a/crates/brk_computer/src/mining/rewards/compute.rs b/crates/brk_computer/src/mining/rewards/compute.rs index 662343e42..fd276177e 100644 --- a/crates/brk_computer/src/mining/rewards/compute.rs +++ b/crates/brk_computer/src/mining/rewards/compute.rs @@ -28,7 +28,12 @@ impl Vecs { prices, &indexer.vecs.transactions.first_tx_index, |_, tx_index| { - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut txout_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut count_cursor = indexes.tx_index.output_count.cursor(); let ti = tx_index.to_usize(); diff --git a/crates/brk_computer/src/mining/rewards/import.rs b/crates/brk_computer/src/mining/rewards/import.rs index 07856be0a..2046c709e 100644 --- a/crates/brk_computer/src/mining/rewards/import.rs +++ b/crates/brk_computer/src/mining/rewards/import.rs @@ -1,6 +1,7 @@ use brk_error::Result; +use brk_indexer::Indexer; use brk_types::Version; -use vecdb::{Database, EagerVec, ImportableVec}; +use vecdb::{AnyVec, Database, EagerVec, ImportableVec}; use super::Vecs; use crate::{ @@ -16,9 +17,15 @@ impl Vecs { pub(crate) fn forced_import( db: &Database, version: Version, + indexer: &Indexer, indexes: &indexes::Vecs, cached_starts: &Windows<&WindowStartVec>, ) -> Result { + let coinbase_version = version + + indexer.vecs.transactions.first_txout_index.version() + + indexes.tx_index.output_count.version() + + indexer.vecs.outputs.value.version(); + let fee_dominance = PercentCumulativeRolling::forced_import(db, "fee_dominance", version, indexes)?; @@ -32,7 +39,7 @@ impl Vecs { coinbase: ValuePerBlockCumulativeRolling::forced_import( db, "coinbase", - version, + coinbase_version, indexes, cached_starts, )?, diff --git a/crates/brk_computer/src/outputs/by_type/compute.rs b/crates/brk_computer/src/outputs/by_type/compute.rs index 332b284a5..c218c82ee 100644 --- a/crates/brk_computer/src/outputs/by_type/compute.rs +++ b/crates/brk_computer/src/outputs/by_type/compute.rs @@ -41,8 +41,13 @@ impl Vecs { let txid_len = indexer.vecs.transactions.txid.len(); let total_txout_len = indexer.vecs.outputs.output_type.len(); - let mut otype_cursor = indexer.vecs.outputs.output_type.cursor(); - let mut fo_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut otype_cursor = indexer.vecs.outputs.output_type.reader().cursor(); + let fo_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut height = skip; walk_blocks( diff --git a/crates/brk_computer/src/outputs/spent/compute.rs b/crates/brk_computer/src/outputs/spent/compute.rs index 82d26b9e8..a56ef0df2 100644 --- a/crates/brk_computer/src/outputs/spent/compute.rs +++ b/crates/brk_computer/src/outputs/spent/compute.rs @@ -18,6 +18,13 @@ impl Vecs { ) -> Result { let starting_lengths = indexer.safe_lengths(); + let dep_version = inputs.spent.txout_index.version() + + indexer.vecs.outputs.first_txout_index.version() + + indexer.vecs.inputs.first_txin_index.version() + + indexer.vecs.outputs.value.version(); + self.txin_index + .validate_computed_version_or_reset(dep_version)?; + let target_height = indexer.vecs.blocks.blockhash.len(); if target_height == 0 { return Ok(exit.lock()); diff --git a/crates/brk_computer/src/pools/mod.rs b/crates/brk_computer/src/pools/mod.rs index 3310591f1..0afd23c85 100644 --- a/crates/brk_computer/src/pools/mod.rs +++ b/crates/brk_computer/src/pools/mod.rs @@ -3,7 +3,10 @@ use std::{collections::BTreeMap, path::Path}; use brk_error::Result; use brk_indexer::Indexer; use brk_traversable::Traversable; -use brk_types::{Addr, AddrBytes, Height, OutputType, PoolSlug, Pools, TxOutIndex, pools}; +use brk_types::{ + Addr, AddrBytes, Height, OutputType, POOL_ATTRIBUTION_VERSION, PoolSlug, Pools, TxOutIndex, + pools, +}; use rayon::prelude::*; use vecdb::{ AnyStoredVec, AnyVec, BytesVec, Database, Exit, ImportableVec, ReadableVec, Rw, StorageMode, @@ -49,7 +52,10 @@ impl Vecs { let db = open_db(parent_path, DB_NAME, 100_000)?; let pools = pools(); - let version = parent_version + Version::new(4) + Version::new(pools.len() as u32); + let version = parent_version + + Version::new(4) + + POOL_ATTRIBUTION_VERSION + + Version::new(pools.len() as u32); let mut major_map = BTreeMap::new(); let mut minor_map = BTreeMap::new(); @@ -121,7 +127,24 @@ impl Vecs { ) -> Result<()> { let starting_height = indexer.safe_lengths().height; - let dep_version = indexer.vecs.blocks.coinbase_tag.version(); + let dep_version: Version = [ + indexer.vecs.blocks.coinbase_tag.version(), + indexer.vecs.transactions.first_tx_index.version(), + indexer.vecs.transactions.first_txout_index.version(), + indexes.tx_index.output_count.version(), + indexer.vecs.outputs.output_type.version(), + indexer.vecs.outputs.type_index.version(), + indexer.vecs.addrs.p2pk65.bytes.version(), + indexer.vecs.addrs.p2pk33.bytes.version(), + indexer.vecs.addrs.p2pkh.bytes.version(), + indexer.vecs.addrs.p2sh.bytes.version(), + indexer.vecs.addrs.p2wpkh.bytes.version(), + indexer.vecs.addrs.p2wsh.bytes.version(), + indexer.vecs.addrs.p2tr.bytes.version(), + indexer.vecs.addrs.p2a.bytes.version(), + ] + .into_iter() + .sum(); let pool_vec_version = self.pool.header().vec_version(); let pool_computed = self.pool.header().computed_version(); let expected = pool_vec_version + dep_version; @@ -167,7 +190,7 @@ impl Vecs { len, |coinbase_tag| -> Result<()> { let tx_index = first_tx_index_cursor.next().unwrap(); - let out_start = first_txout_index.get(tx_index.to_usize()); + let out_start = first_txout_index.get(tx_index); let ti = tx_index.to_usize(); output_count_cursor.advance(ti - output_count_cursor.position()); @@ -176,17 +199,17 @@ impl Vecs { let pool = (*out_start..(*out_start + *output_count_val)) .map(TxOutIndex::from) .find_map(|txout_index| { - let ot = output_type.get(txout_index.to_usize()); - let ti = usize::from(type_index.get(txout_index.to_usize())); + let ot = output_type.get(txout_index); + let ti = usize::from(type_index.get(txout_index)); match ot { - OutputType::P2PK65 => Some(AddrBytes::from(p2pk65.get(ti))), - OutputType::P2PK33 => Some(AddrBytes::from(p2pk33.get(ti))), - OutputType::P2PKH => Some(AddrBytes::from(p2pkh.get(ti))), - OutputType::P2SH => Some(AddrBytes::from(p2sh.get(ti))), - OutputType::P2WPKH => Some(AddrBytes::from(p2wpkh.get(ti))), - OutputType::P2WSH => Some(AddrBytes::from(p2wsh.get(ti))), - OutputType::P2TR => Some(AddrBytes::from(p2tr.get(ti))), - OutputType::P2A => Some(AddrBytes::from(p2a.get(ti))), + OutputType::P2PK65 => Some(AddrBytes::from(p2pk65.get_at(ti))), + OutputType::P2PK33 => Some(AddrBytes::from(p2pk33.get_at(ti))), + OutputType::P2PKH => Some(AddrBytes::from(p2pkh.get_at(ti))), + OutputType::P2SH => Some(AddrBytes::from(p2sh.get_at(ti))), + OutputType::P2WPKH => Some(AddrBytes::from(p2wpkh.get_at(ti))), + OutputType::P2WSH => Some(AddrBytes::from(p2wsh.get_at(ti))), + OutputType::P2TR => Some(AddrBytes::from(p2tr.get_at(ti))), + OutputType::P2A => Some(AddrBytes::from(p2a.get_at(ti))), _ => None, } .map(|bytes| Addr::try_from(&bytes).unwrap()) diff --git a/crates/brk_computer/src/pools/pool_heights.rs b/crates/brk_computer/src/pools/pool_heights.rs index bad5975b9..4b7e3f676 100644 --- a/crates/brk_computer/src/pools/pool_heights.rs +++ b/crates/brk_computer/src/pools/pool_heights.rs @@ -14,7 +14,9 @@ impl PoolHeights { let mut map: FxHashMap> = FxHashMap::default(); let reader = pool.reader(); for h in 0..len { - map.entry(reader.get(h)).or_default().push(Height::from(h)); + map.entry(reader.get_at(h)) + .or_default() + .push(Height::from(h)); } Self(Arc::new(RwLock::new(map))) } diff --git a/crates/brk_computer/src/price/compute.rs b/crates/brk_computer/src/price/compute.rs index e7ef5c37b..7cf32c68d 100644 --- a/crates/brk_computer/src/price/compute.rs +++ b/crates/brk_computer/src/price/compute.rs @@ -63,8 +63,16 @@ impl Vecs { fn compute_prices(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { let starting_height = indexer.safe_lengths().height; - let source_version = - indexer.vecs.outputs.value.version() + indexer.vecs.outputs.output_type.version(); + let source_version = [ + indexer.vecs.transactions.txid.version(), + indexer.vecs.transactions.first_tx_index.version(), + indexer.vecs.outputs.first_txout_index.version(), + indexer.vecs.transactions.first_txout_index.version(), + indexer.vecs.outputs.value.version(), + indexer.vecs.outputs.output_type.version(), + ] + .into_iter() + .sum(); self.spot .cents .height diff --git a/crates/brk_computer/src/transactions/patterns/compute.rs b/crates/brk_computer/src/transactions/patterns/compute.rs index a7e5f2eb3..5f23fae96 100644 --- a/crates/brk_computer/src/transactions/patterns/compute.rs +++ b/crates/brk_computer/src/transactions/patterns/compute.rs @@ -101,9 +101,9 @@ impl Vecs { let mut input_value = spent.value.cursor(); let mut input_type = indexer.vecs.inputs.output_type.cursor(); let mut input_type_index = indexer.vecs.inputs.type_index.cursor(); - let mut output_value = indexer.vecs.outputs.value.cursor(); - let mut output_type = indexer.vecs.outputs.output_type.cursor(); - let mut output_type_index = indexer.vecs.outputs.type_index.cursor(); + let mut output_value = indexer.vecs.outputs.value.reader().cursor(); + let mut output_type = indexer.vecs.outputs.output_type.reader().cursor(); + let mut output_type_index = indexer.vecs.outputs.type_index.reader().cursor(); let mut has_op_return = features.has_op_return.cursor(); let mut has_inscription = features.has_inscription.cursor(); let mut tx_count = indexes.height.tx_index_count.cursor(); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index ebc54dd4a..92c9bb043 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -12,6 +12,7 @@ use brk_reader::{Reader, XORBytes}; use brk_rpc::Client; use brk_types::{BlockHash, Height}; use fjall::PersistMode; +use rayon::prelude::*; use tracing::{debug, error, info}; use vecdb::{ Exit, RawDBError, ReadOnlyClone, ReadableVec, Ro, Rw, StorageMode, WritableVec, unlikely, @@ -332,9 +333,9 @@ impl Indexer { if !tasks.is_empty() { let i = Instant::now(); - for task in tasks { - task().map_err(vecdb::RawDBError::other)?; - } + tasks + .into_par_iter() + .try_for_each(|task| task().map_err(vecdb::RawDBError::other))?; debug!("Stores committed in {:?}", i.elapsed()); let i = Instant::now(); diff --git a/crates/brk_indexer/src/processor/transaction/mod.rs b/crates/brk_indexer/src/processor/transaction/mod.rs index 7065308a3..006cc4a56 100644 --- a/crates/brk_indexer/src/processor/transaction/mod.rs +++ b/crates/brk_indexer/src/processor/transaction/mod.rs @@ -78,7 +78,7 @@ impl<'a> BlockProcessor<'a> { .vecs .transactions .txid - .get_pushed_or_read(prev_tx_index, &self.readers.txid) + .get_append_only(prev_tx_index, &self.readers.txid) .ok_or(Error::Internal("Missing txid for tx_index")) .inspect_err(|_| { error!(?tx_index, len, "Missing txid for tx_index"); @@ -104,10 +104,10 @@ impl<'a> BlockProcessor<'a> { &mut self, txs: Vec, mut txouts: Vec, - txins: Vec, + txins: &[InputSource], addresses: &mut BlockAddresses, ) -> Result<()> { - let transaction_analyses = self.analyze_transactions(&txs, &txins, &txouts); + let transaction_analyses = self.analyze_transactions(&txs, txins, &txouts); let lengths = &mut *self.lengths; let base_tx_index = lengths.tx_index; let base_txin_index = lengths.txin_index; diff --git a/crates/brk_indexer/src/processor/txin/mod.rs b/crates/brk_indexer/src/processor/txin/mod.rs index 742ace26d..8abe367ea 100644 --- a/crates/brk_indexer/src/processor/txin/mod.rs +++ b/crates/brk_indexer/src/processor/txin/mod.rs @@ -18,11 +18,11 @@ use super::{BlockProcessor, transaction::ComputedTx, txout::ProcessedOutput}; use crate::InputsVecs; impl<'a> BlockProcessor<'a> { - pub(crate) fn process_inputs( + pub(crate) fn process_inputs<'b>( &self, txs: &[ComputedTx], - resolver: &mut InputResolver, - ) -> Result> { + resolver: &'b mut InputResolver, + ) -> Result<&'b [InputSource]> { resolver.resolve(self, txs) } } @@ -36,7 +36,7 @@ pub(super) fn finalize_inputs( inputs: &mut InputsVecs, addr_tx_index_stores: &mut ByAddrType>, addr_outpoint_stores: &mut ByAddrType>, - txins: Vec, + txins: &[InputSource], txouts: &[ProcessedOutput], ) -> Result<()> { let mut input_offset = 0; diff --git a/crates/brk_indexer/src/processor/txin/resolver.rs b/crates/brk_indexer/src/processor/txin/resolver.rs index 21090e43e..f272b87b4 100644 --- a/crates/brk_indexer/src/processor/txin/resolver.rs +++ b/crates/brk_indexer/src/processor/txin/resolver.rs @@ -12,13 +12,15 @@ use vecdb::unlikely; use super::InputSource; use crate::processor::{BlockProcessor, transaction::ComputedTx}; +const PARALLEL_PARENT_READ_THRESHOLD: usize = 1_000; + #[derive(Default)] pub(crate) struct InputResolver { - same_block_transactions: FxHashMap, - previous_parent_indexes: FxHashMap, - previous_parents: Vec, + parent_locations: FxHashMap, + previous_parent_prefixes: Vec, inputs: Vec, reads: ReadBatch, + resolved: Vec, } impl InputResolver { @@ -26,65 +28,78 @@ impl InputResolver { &mut self, processor: &BlockProcessor<'_>, txs: &[ComputedTx<'_>], - ) -> Result> { + ) -> Result<&[InputSource]> { self.prepare(txs, processor.lengths.tx_index); self.reads.resolve( processor, - &self.previous_parents, + &self.previous_parent_prefixes, &self.inputs, processor.lengths.tx_index, )?; let tracks_executed_legacy_sigops = processor.tracks_executed_legacy_sigops(); let reads = &self.reads; + let inputs = &self.inputs; - self.inputs - .par_iter() - .enumerate() - .map(|(input_index, input)| match *input { - UnresolvedInput::Coinbase => Ok(InputSource::Coinbase), - UnresolvedInput::SameBlock { - outpoint, - txout_offset, - } => Ok(InputSource::SameBlock { - outpoint, - txout_offset, - }), - UnresolvedInput::PreviousBlock { parent_index, vout } => { - let parent = reads.parent(parent_index); - let outpoint = OutPoint::new(parent.tx_index, vout); - let (output_type, type_index) = reads.output(input_index); - - let legacy_sigops = if tracks_executed_legacy_sigops { - processor - .vecs - .scripts - .legacy_sigops(output_type, type_index, &processor.readers.scripts) - .ok_or(Error::Internal("Missing legacy_sigops"))? - } else { - SigOps::ZERO - }; - - Ok(InputSource::PreviousBlock { + self.resolved.clear(); + self.resolved.resize(inputs.len(), InputSource::Coinbase); + self.resolved.par_iter_mut().enumerate().try_for_each( + |(input_index, resolved)| -> Result<()> { + match inputs[input_index] { + UnresolvedInput::Coinbase => { + *resolved = InputSource::Coinbase; + Ok(()) + } + UnresolvedInput::SameBlock { outpoint, - output_type, - legacy_sigops, - type_index, - }) + txout_offset, + } => { + *resolved = InputSource::SameBlock { + outpoint, + txout_offset, + }; + Ok(()) + } + UnresolvedInput::PreviousBlock { parent_index, vout } => { + let parent = reads.parent(parent_index); + let outpoint = OutPoint::new(parent.tx_index, vout); + let (output_type, type_index) = reads.output(input_index); + + let legacy_sigops = if tracks_executed_legacy_sigops { + processor + .vecs + .scripts + .legacy_sigops(output_type, type_index, &processor.readers.scripts) + .ok_or(Error::Internal("Missing legacy_sigops"))? + } else { + SigOps::ZERO + }; + + *resolved = InputSource::PreviousBlock { + outpoint, + output_type, + legacy_sigops, + type_index, + }; + Ok(()) + } } - }) - .collect() + }, + )?; + + Ok(&self.resolved) } fn prepare(&mut self, txs: &[ComputedTx<'_>], block_first_tx_index: TxIndex) { - self.same_block_transactions.clear(); - self.previous_parent_indexes.clear(); - self.previous_parents.clear(); + self.parent_locations.clear(); + self.previous_parent_prefixes.clear(); self.inputs.clear(); - self.same_block_transactions.reserve(txs.len()); - self.same_block_transactions - .extend(txs.iter().map(|tx| (tx.txid_prefix(), tx.tx_index))); + self.parent_locations.reserve(txs.len()); + self.parent_locations.extend( + txs.iter() + .map(|tx| (tx.txid_prefix(), ParentLocation::SameBlock(tx.tx_index))), + ); let total_inputs = txs.iter().map(|tx| tx.tx.input.len()).sum(); self.inputs.reserve(total_inputs); @@ -101,22 +116,25 @@ impl InputResolver { let txid_prefix = TxidPrefix::from(&txid); let vout = Vout::from(previous_output.vout); - if let Some(&tx_index) = self.same_block_transactions.get(&txid_prefix) { - let block_tx_index = usize::from(tx_index) - usize::from(block_first_tx_index); - self.inputs.push(UnresolvedInput::SameBlock { - outpoint: OutPoint::new(tx_index, vout), - txout_offset: txs[block_tx_index].txout_offset(vout), - }); - continue; - } - - let parent_index = match self.previous_parent_indexes.entry(txid_prefix) { - Entry::Occupied(entry) => *entry.get(), + let parent_index = match self.parent_locations.entry(txid_prefix) { + Entry::Occupied(entry) => match *entry.get() { + ParentLocation::SameBlock(tx_index) => { + let block_tx_index = + usize::from(tx_index) - usize::from(block_first_tx_index); + self.inputs.push(UnresolvedInput::SameBlock { + outpoint: OutPoint::new(tx_index, vout), + txout_offset: txs[block_tx_index].txout_offset(vout), + }); + continue; + } + ParentLocation::Previous(parent_index) => parent_index.to_usize(), + }, Entry::Vacant(entry) => { - let parent_index = self.previous_parents.len(); - entry.insert(parent_index); - self.previous_parents - .push(PreviousParent { txid, txid_prefix }); + let parent_index = self.previous_parent_prefixes.len(); + entry.insert(ParentLocation::Previous(PreviousParentIndex::new( + parent_index, + ))); + self.previous_parent_prefixes.push(txid_prefix); parent_index } }; @@ -129,14 +147,32 @@ impl InputResolver { } #[derive(Clone, Copy)] -struct PreviousParent { - txid: Txid, - txid_prefix: TxidPrefix, +enum ParentLocation { + SameBlock(TxIndex), + Previous(PreviousParentIndex), } +#[derive(Clone, Copy)] +struct PreviousParentIndex(u32); + +impl PreviousParentIndex { + fn new(index: usize) -> Self { + Self( + u32::try_from(index) + .expect("number of unique previous parents in a block must fit in u32"), + ) + } + + #[inline] + fn to_usize(self) -> usize { + self.0 as usize + } +} + +const _: () = assert!(size_of::() == 8); + #[derive(Clone, Copy)] struct ParentRead { - original_index: usize, tx_index: TxIndex, first_txout_index: TxOutIndex, } @@ -150,9 +186,7 @@ struct OutputRead { #[derive(Default)] struct ReadBatch { parents: Vec, - parent_positions: Vec, outputs: Vec, - output_positions: Vec, output_types: Vec, type_indices: Vec, } @@ -161,11 +195,11 @@ impl ReadBatch { fn resolve( &mut self, processor: &BlockProcessor<'_>, - previous_parents: &[PreviousParent], + previous_parent_prefixes: &[TxidPrefix], inputs: &[UnresolvedInput], current_tx_index: TxIndex, ) -> Result<()> { - self.resolve_parents(processor, previous_parents, current_tx_index)?; + self.resolve_parents(processor, previous_parent_prefixes, current_tx_index)?; self.prepare_outputs(inputs); self.read_outputs(processor) } @@ -173,61 +207,66 @@ impl ReadBatch { fn resolve_parents( &mut self, processor: &BlockProcessor<'_>, - previous_parents: &[PreviousParent], + previous_parent_prefixes: &[TxidPrefix], current_tx_index: TxIndex, ) -> Result<()> { + let parallel_raw_reads = previous_parent_prefixes.len() >= PARALLEL_PARENT_READ_THRESHOLD; + self.parents.clear(); - self.parents.extend( - (0..previous_parents.len()).map(|original_index| ParentRead { - original_index, + self.parents.resize( + previous_parent_prefixes.len(), + ParentRead { tx_index: TxIndex::default(), first_txout_index: TxOutIndex::default(), - }), + }, ); self.parents .par_iter_mut() + .zip(previous_parent_prefixes.par_iter()) .try_for_each(|read| { - let parent = &previous_parents[read.original_index]; + let (read, txid_prefix) = read; let store_result = processor .stores .txid_prefix_to_tx_index - .get(&parent.txid_prefix)? + .get(txid_prefix)? .map(|value| *value); let tx_index = match store_result { Some(tx_index) if tx_index < current_tx_index => tx_index, _ => { error!( - "UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}", - parent.txid, - parent.txid_prefix, - store_result, - current_tx_index + "UnknownTxid: prefix={:?}, store_result={:?}, current_tx_index={:?}", + txid_prefix, store_result, current_tx_index ); return Err(Error::UnknownTxid); } }; read.tx_index = tx_index; + if parallel_raw_reads { + read.first_txout_index = processor + .vecs + .transactions + .first_txout_index + .get_append_only(tx_index, &processor.readers.tx_index_to_first_txout_index) + .ok_or(Error::Internal("Missing txout_index"))?; + } Ok(()) })?; - self.parents.sort_unstable_by_key(|read| read.tx_index); - self.parent_positions.clear(); - self.parent_positions.resize(self.parents.len(), 0); - - for (position, read) in self.parents.iter_mut().enumerate() { - self.parent_positions[read.original_index] = position; - read.first_txout_index = processor - .vecs - .transactions - .first_txout_index - .get_pushed_or_read( - read.tx_index, - &processor.readers.tx_index_to_first_txout_index, - ) - .ok_or(Error::Internal("Missing txout_index"))?; + if !parallel_raw_reads { + for read in &mut self.parents { + read.first_txout_index = processor + .vecs + .transactions + .first_txout_index + .get_append_only( + read.tx_index, + &processor.readers.tx_index_to_first_txout_index, + ) + .ok_or(Error::Internal("Missing txout_index"))?; + } } Ok(()) @@ -247,55 +286,47 @@ impl ReadBatch { } } - self.outputs.sort_unstable_by_key(|read| read.txout_index); - self.output_positions.clear(); - self.output_positions.resize(inputs.len(), 0); - - for (position, read) in self.outputs.iter().enumerate() { - self.output_positions[read.input_index] = position; - } + self.output_types.clear(); + self.output_types.resize(inputs.len(), OutputType::Unknown); + self.type_indices.clear(); + self.type_indices.resize(inputs.len(), TypeIndex::default()); } fn read_outputs(&mut self, processor: &BlockProcessor<'_>) -> Result<()> { - self.output_types.clear(); - self.output_types.reserve(self.outputs.len()); - self.type_indices.clear(); - self.type_indices.reserve(self.outputs.len()); - let outputs = &self.outputs; + if outputs.is_empty() { + return Ok(()); + } + let output_types = &mut self.output_types; let type_indices = &mut self.type_indices; let (output_types_result, type_indices_result) = rayon::join( || -> Result<()> { for read in outputs { - output_types.push( - processor - .vecs - .outputs - .output_type - .get_pushed_or_read( - read.txout_index, - &processor.readers.txout_index_to_output_type, - ) - .ok_or(Error::Internal("Missing output_type"))?, - ); + output_types[read.input_index] = processor + .vecs + .outputs + .output_type + .get_append_only( + read.txout_index, + &processor.readers.txout_index_to_output_type, + ) + .ok_or(Error::Internal("Missing output_type"))?; } Ok(()) }, || -> Result<()> { for read in outputs { - type_indices.push( - processor - .vecs - .outputs - .type_index - .get_pushed_or_read( - read.txout_index, - &processor.readers.txout_index_to_type_index, - ) - .ok_or(Error::Internal("Missing type_index"))?, - ); + type_indices[read.input_index] = processor + .vecs + .outputs + .type_index + .get_append_only( + read.txout_index, + &processor.readers.txout_index_to_type_index, + ) + .ok_or(Error::Internal("Missing type_index"))?; } Ok(()) }, @@ -306,12 +337,14 @@ impl ReadBatch { } fn parent(&self, original_index: usize) -> ParentRead { - self.parents[self.parent_positions[original_index]] + self.parents[original_index] } fn output(&self, input_index: usize) -> (OutputType, TypeIndex) { - let position = self.output_positions[input_index]; - (self.output_types[position], self.type_indices[position]) + ( + self.output_types[input_index], + self.type_indices[input_index], + ) } } diff --git a/crates/brk_indexer/src/processor/txin/source.rs b/crates/brk_indexer/src/processor/txin/source.rs index 87fdecd17..833d376f8 100644 --- a/crates/brk_indexer/src/processor/txin/source.rs +++ b/crates/brk_indexer/src/processor/txin/source.rs @@ -1,6 +1,6 @@ use brk_types::{OutPoint, OutputType, SigOps, TypeIndex}; -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub(crate) enum InputSource { Coinbase, PreviousBlock { diff --git a/crates/brk_indexer/src/readers.rs b/crates/brk_indexer/src/readers.rs index c5c40271a..1c035e4ee 100644 --- a/crates/brk_indexer/src/readers.rs +++ b/crates/brk_indexer/src/readers.rs @@ -24,14 +24,14 @@ impl AddrReaders { pub fn script_pubkey(&self, output_type: OutputType, type_index: TypeIndex) -> ScriptBuf { let idx = usize::from(type_index); let bytes: Option = match output_type { - OutputType::P2PK65 => self.p2pk65.try_get(idx).map(Into::into), - OutputType::P2PK33 => self.p2pk33.try_get(idx).map(Into::into), - OutputType::P2PKH => self.p2pkh.try_get(idx).map(Into::into), - OutputType::P2SH => self.p2sh.try_get(idx).map(Into::into), - OutputType::P2WPKH => self.p2wpkh.try_get(idx).map(Into::into), - OutputType::P2WSH => self.p2wsh.try_get(idx).map(Into::into), - OutputType::P2TR => self.p2tr.try_get(idx).map(Into::into), - OutputType::P2A => self.p2a.try_get(idx).map(Into::into), + OutputType::P2PK65 => self.p2pk65.try_get_at(idx).map(Into::into), + OutputType::P2PK33 => self.p2pk33.try_get_at(idx).map(Into::into), + OutputType::P2PKH => self.p2pkh.try_get_at(idx).map(Into::into), + OutputType::P2SH => self.p2sh.try_get_at(idx).map(Into::into), + OutputType::P2WPKH => self.p2wpkh.try_get_at(idx).map(Into::into), + OutputType::P2WSH => self.p2wsh.try_get_at(idx).map(Into::into), + OutputType::P2TR => self.p2tr.try_get_at(idx).map(Into::into), + OutputType::P2A => self.p2a.try_get_at(idx).map(Into::into), _ => None, }; bytes.map(|b| b.to_script_pubkey()).unwrap_or_default() diff --git a/crates/brk_indexer/src/stores.rs b/crates/brk_indexer/src/stores.rs index a6d1f6b09..48e7a530d 100644 --- a/crates/brk_indexer/src/stores.rs +++ b/crates/brk_indexer/src/stores.rs @@ -332,13 +332,13 @@ impl Stores { .collect_range_at(rollback_start, rollback_end); for (i, txout_index) in (rollback_start..rollback_end).enumerate() { - let output_type = txout_index_to_output_type_reader.get(txout_index); + let output_type = txout_index_to_output_type_reader.get_at(txout_index); if !output_type.is_addr() { continue; } let addr_type = output_type; - let addr_index = txout_index_to_type_index_reader.get(txout_index); + let addr_index = txout_index_to_type_index_reader.get_at(txout_index); let tx_index = tx_indexes[i]; addr_index_tx_index_to_remove.insert((addr_type, addr_index, tx_index)); @@ -346,7 +346,7 @@ impl Stores { let vout = Vout::from( txout_index - tx_index_to_first_txout_index_reader - .get(tx_index.to_usize()) + .get(tx_index) .to_usize(), ); let outpoint = OutPoint::new(tx_index, vout); @@ -371,12 +371,11 @@ impl Stores { let output_tx_index = outpoint.tx_index(); let vout = outpoint.vout(); - let txout_index = - tx_index_to_first_txout_index_reader.get(output_tx_index.to_usize()) + vout; + let txout_index = tx_index_to_first_txout_index_reader.get(output_tx_index) + vout; if txout_index < starting_lengths.txout_index { - let output_type = txout_index_to_output_type_reader.get(txout_index.to_usize()); - let type_index = txout_index_to_type_index_reader.get(txout_index.to_usize()); + let output_type = txout_index_to_output_type_reader.get(txout_index); + let type_index = txout_index_to_type_index_reader.get(txout_index); Some((outpoint, output_type, type_index, spending_tx_index)) } else { None diff --git a/crates/brk_indexer/src/vecs/addrs.rs b/crates/brk_indexer/src/vecs/addrs.rs index aacd535ed..73a7556b7 100644 --- a/crates/brk_indexer/src/vecs/addrs.rs +++ b/crates/brk_indexer/src/vecs/addrs.rs @@ -233,42 +233,42 @@ impl AddrsVecs { OutputType::P2PK65 => self .p2pk65 .bytes - .get_pushed_or_read(type_index.into(), &readers.p2pk65) + .get_append_only(type_index.into(), &readers.p2pk65) .map(AddrBytes::from), OutputType::P2PK33 => self .p2pk33 .bytes - .get_pushed_or_read(type_index.into(), &readers.p2pk33) + .get_append_only(type_index.into(), &readers.p2pk33) .map(AddrBytes::from), OutputType::P2PKH => self .p2pkh .bytes - .get_pushed_or_read(type_index.into(), &readers.p2pkh) + .get_append_only(type_index.into(), &readers.p2pkh) .map(AddrBytes::from), OutputType::P2SH => self .p2sh .bytes - .get_pushed_or_read(type_index.into(), &readers.p2sh) + .get_append_only(type_index.into(), &readers.p2sh) .map(AddrBytes::from), OutputType::P2WPKH => self .p2wpkh .bytes - .get_pushed_or_read(type_index.into(), &readers.p2wpkh) + .get_append_only(type_index.into(), &readers.p2wpkh) .map(AddrBytes::from), OutputType::P2WSH => self .p2wsh .bytes - .get_pushed_or_read(type_index.into(), &readers.p2wsh) + .get_append_only(type_index.into(), &readers.p2wsh) .map(AddrBytes::from), OutputType::P2TR => self .p2tr .bytes - .get_pushed_or_read(type_index.into(), &readers.p2tr) + .get_append_only(type_index.into(), &readers.p2tr) .map(AddrBytes::from), OutputType::P2A => self .p2a .bytes - .get_pushed_or_read(type_index.into(), &readers.p2a) + .get_append_only(type_index.into(), &readers.p2a) .map(AddrBytes::from), _ => unreachable!("get_bytes_by_type called with non-address type"), } @@ -302,7 +302,7 @@ impl AddrsVecs { Some(mut index) => { let reader = $addr.bytes.reader(); Ok(Box::new(std::iter::from_fn(move || { - reader.try_get(index.to_usize()).map(|typedbytes| { + reader.try_get(index).map(|typedbytes| { let bytes = AddrBytes::from(typedbytes); index.increment(); AddrHash::from(&bytes) diff --git a/crates/brk_indexer/src/vecs/scripts.rs b/crates/brk_indexer/src/vecs/scripts.rs index f9c3eb079..b82fa741c 100644 --- a/crates/brk_indexer/src/vecs/scripts.rs +++ b/crates/brk_indexer/src/vecs/scripts.rs @@ -153,11 +153,11 @@ impl ScriptsVecs { OutputType::P2MS => self .p2ms .legacy_sigops - .get_pushed_or_read(type_index.into(), &readers.p2ms_legacy_sigops), + .get_append_only(type_index.into(), &readers.p2ms_legacy_sigops), OutputType::Unknown => self .unknown .legacy_sigops - .get_pushed_or_read(type_index.into(), &readers.unknown_legacy_sigops), + .get_append_only(type_index.into(), &readers.unknown_legacy_sigops), _ => Some(SigOps::ZERO), } } diff --git a/crates/brk_oracle/examples/determinism.rs b/crates/brk_oracle/examples/determinism.rs index 4be2c3185..1145c10e8 100644 --- a/crates/brk_oracle/examples/determinism.rs +++ b/crates/brk_oracle/examples/determinism.rs @@ -79,7 +79,12 @@ fn main() { let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); let out_first: Vec = indexer.vecs.outputs.first_txout_index.collect(); - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut txout_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut blocks: Vec = Vec::with_capacity(end_height - load_start); for h in load_start..end_height { diff --git a/crates/brk_oracle/examples/dump_hist.rs b/crates/brk_oracle/examples/dump_hist.rs index 3d9650057..fbcfa1e5f 100644 --- a/crates/brk_oracle/examples/dump_hist.rs +++ b/crates/brk_oracle/examples/dump_hist.rs @@ -55,7 +55,12 @@ fn main() { let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); let out_first: Vec = indexer.vecs.outputs.first_txout_index.collect(); - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut txout_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut tx_starts: Vec = Vec::new(); let out_path = format!("{out_dir}/oracle_outputs_{start}_{end}.csv"); diff --git a/crates/brk_oracle/examples/experiment.rs b/crates/brk_oracle/examples/experiment.rs index 7c4ece357..df86a0eee 100644 --- a/crates/brk_oracle/examples/experiment.rs +++ b/crates/brk_oracle/examples/experiment.rs @@ -534,7 +534,12 @@ fn main() { let total_outputs = indexer.vecs.outputs.value.len(); let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); let out_first: Vec = indexer.vecs.outputs.first_txout_index.collect(); - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut txout_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut tx_starts: Vec = Vec::new(); let mut values: Vec = Vec::new(); let mut output_types: Vec = Vec::new(); diff --git a/crates/brk_oracle/examples/report.rs b/crates/brk_oracle/examples/report.rs index 497a77645..f1734d159 100644 --- a/crates/brk_oracle/examples/report.rs +++ b/crates/brk_oracle/examples/report.rs @@ -180,7 +180,12 @@ fn main() { // large, so the tx-indexed first_txout_index is read through a forward cursor. let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); let out_first: Vec = indexer.vecs.outputs.first_txout_index.collect(); - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut txout_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut tx_starts: Vec = Vec::new(); let mut year_stats: Vec = Vec::new(); diff --git a/crates/brk_oracle/examples/report_from.rs b/crates/brk_oracle/examples/report_from.rs index 13504d2ac..25af8bf8f 100644 --- a/crates/brk_oracle/examples/report_from.rs +++ b/crates/brk_oracle/examples/report_from.rs @@ -842,7 +842,12 @@ fn main() { // large, so the tx-indexed first_txout_index is read through a forward cursor. let first_tx_index: Vec = indexer.vecs.transactions.first_tx_index.collect(); let out_first: Vec = indexer.vecs.outputs.first_txout_index.collect(); - let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut txout_cursor = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut tx_starts: Vec = Vec::new(); let mut year_stats: Vec = Vec::new(); diff --git a/crates/brk_query/Cargo.toml b/crates/brk_query/Cargo.toml index b150c0173..afacb4604 100644 --- a/crates/brk_query/Cargo.toml +++ b/crates/brk_query/Cargo.toml @@ -25,8 +25,7 @@ brk_types = { workspace = true } derive_more = { workspace = true } jiff = { workspace = true } parking_lot = { workspace = true } -# quickmatch = { path = "../../../quickmatch" } -quickmatch = "0.5.0" +quickmatch = { workspace = true } rustc-hash = { workspace = true } smallvec = { workspace = true } tokio = { workspace = true, optional = true } diff --git a/crates/brk_query/src/impl/addr/stats.rs b/crates/brk_query/src/impl/addr/stats.rs index 7fa1b1b39..265660ee7 100644 --- a/crates/brk_query/src/impl/addr/stats.rs +++ b/crates/brk_query/src/impl/addr/stats.rs @@ -5,6 +5,7 @@ use brk_types::{ Addr, AddrBytes, AddrChainStats, AddrHash, AddrStats, AnyAddrDataIndexEnum, Dollars, OutputType, TypeIndex, }; +use vecdb::ReadableVec; use crate::Query; @@ -40,8 +41,8 @@ impl Query { .distribution .addrs_data .funded - .reader() - .get(usize::from(index)); + .collect_one(index) + .expect("funded address data index should be in bounds"); let price = data.realized_price().to_dollars(); (data, price) } @@ -50,8 +51,8 @@ impl Query { .distribution .addrs_data .empty - .reader() - .get(usize::from(index)) + .collect_one(index) + .expect("empty address data index should be in bounds") .into(); (data, Dollars::default()) } diff --git a/crates/brk_query/src/impl/addr/txs.rs b/crates/brk_query/src/impl/addr/txs.rs index 6d0598331..4eb09aad2 100644 --- a/crates/brk_query/src/impl/addr/txs.rs +++ b/crates/brk_query/src/impl/addr/txs.rs @@ -1,6 +1,5 @@ use brk_error::{OptionData, Result}; use brk_types::{Addr, AddrIndexTxIndex, Transaction, TxIndex, Txid, Unit}; -use vecdb::VecIndex; use crate::Query; @@ -25,7 +24,7 @@ impl Query { let txid_reader = self.indexer().vecs.transactions.txid.reader(); Ok(txindices .into_iter() - .map(|tx_index| txid_reader.get(tx_index.to_usize())) + .map(|tx_index| txid_reader.get(tx_index)) .collect()) } diff --git a/crates/brk_query/src/impl/addr/utxos.rs b/crates/brk_query/src/impl/addr/utxos.rs index 0aeb8f37f..49635c378 100644 --- a/crates/brk_query/src/impl/addr/utxos.rs +++ b/crates/brk_query/src/impl/addr/utxos.rs @@ -1,6 +1,5 @@ use brk_error::{Error, OptionData, Result}; use brk_types::{Addr, AddrIndexOutPoint, Height, TxIndex, TxStatus, Unit, Utxo, Vout}; -use vecdb::VecIndex; use crate::Query; @@ -36,9 +35,9 @@ impl Query { let mut utxos = Vec::with_capacity(outpoints.len()); for (tx_index, vout) in outpoints { - let txid = txid_reader.get(tx_index.to_usize()); - let first_txout_index = first_txout_index_reader.get(tx_index.to_usize()); - let value = value_reader.get(usize::from(first_txout_index + vout)); + let txid = txid_reader.get(tx_index); + let first_txout_index = first_txout_index_reader.get(tx_index); + let value = value_reader.get(first_txout_index + vout); let height = self.confirmed_status_height(tx_index)?; let status = if let Some((h, ref s)) = cached_status diff --git a/crates/brk_query/src/impl/block/txs.rs b/crates/brk_query/src/impl/block/txs.rs index 234f00cc4..f7be3fb85 100644 --- a/crates/brk_query/src/impl/block/txs.rs +++ b/crates/brk_query/src/impl/block/txs.rs @@ -88,8 +88,7 @@ impl Query { .vecs .transactions .txid - .reader() - .try_get(first + index) + .collect_one_at(first + index) .ok_or(Error::Internal( "block_txid_at_index_by_height: txid index past data", )) @@ -125,7 +124,7 @@ impl Query { // ── Phase 1: Decode all transactions, collect outpoints ───────── - let mut txid_cursor = indexer.vecs.transactions.txid.cursor(); + let txid_cursor = indexer.vecs.transactions.txid.reader().cursor(); let mut total_size_cursor = indexer.vecs.transactions.total_size.cursor(); let mut sigops_cursor = indexer.vecs.transactions.total_sigop_cost.cursor(); let mut first_txin_cursor = indexer.vecs.transactions.first_txin_index.cursor(); diff --git a/crates/brk_query/src/impl/cpfp/confirmed.rs b/crates/brk_query/src/impl/cpfp/confirmed.rs index ac067c29a..3585b4198 100644 --- a/crates/brk_query/src/impl/cpfp/confirmed.rs +++ b/crates/brk_query/src/impl/cpfp/confirmed.rs @@ -66,7 +66,7 @@ impl Query { let position = index.to_usize(); let weight = weight.get(position).data()?; Ok(Member { - txid: txid.get(position), + txid: txid.get(*index), fee: fee.get(position).data()?, weight, vsize: VSize::from(weight), @@ -88,7 +88,7 @@ impl Query { .map(|index| { let position = index.to_usize(); Ok(CpfpEntry { - txid: txid.get(position), + txid: txid.get(*index), fee: fee.get(position).data()?, weight: weight.get(position).data()?, }) @@ -112,9 +112,14 @@ impl Query { let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor(); let mut input_count = computer.indexes.tx_index.input_count.cursor(); let mut outpoint = indexer.vecs.inputs.outpoint.cursor(); - let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor(); + let first_txout = indexer + .vecs + .transactions + .first_txout_index + .reader() + .cursor(); let mut output_count = computer.indexes.tx_index.output_count.cursor(); - let mut spent = computer.outputs.spent.txin_index.cursor(); + let spent = computer.outputs.spent.txin_index.reader().cursor(); let mut spending_tx = indexer.vecs.inputs.tx_index.cursor(); let mut parents_of = |tx: TxIndex| -> Result> { diff --git a/crates/brk_query/src/impl/mempool.rs b/crates/brk_query/src/impl/mempool.rs index d69a8ca25..d03a9af76 100644 --- a/crates/brk_query/src/impl/mempool.rs +++ b/crates/brk_query/src/impl/mempool.rs @@ -67,16 +67,14 @@ impl Query { if prev_tx_index >= safe.tx_index { return None; } - let first_txout: TxOutIndex = - first_txout_reader.try_get(usize::from(prev_tx_index))?; + let first_txout: TxOutIndex = first_txout_reader.try_get(prev_tx_index)?; let txout = first_txout + *vout; if txout >= safe.txout_index { return None; } - let txout_idx = usize::from(txout); - let output_type: OutputType = output_type_reader.try_get(txout_idx)?; - let type_index: TypeIndex = type_index_reader.try_get(txout_idx)?; - let value: Sats = value_reader.try_get(txout_idx)?; + let output_type: OutputType = output_type_reader.try_get(txout)?; + let type_index: TypeIndex = type_index_reader.try_get(txout)?; + let value: Sats = value_reader.try_get(txout)?; let script_pubkey = addr_readers.script_pubkey(output_type, type_index); Some(((*prev_txid, *vout), TxOut::from((script_pubkey, value)))) }) diff --git a/crates/brk_query/src/impl/tx.rs b/crates/brk_query/src/impl/tx.rs index b510603dd..ff9133341 100644 --- a/crates/brk_query/src/impl/tx.rs +++ b/crates/brk_query/src/impl/tx.rs @@ -205,8 +205,8 @@ impl Query { .outputs .spent .txin_index - .reader() - .get(usize::from(txout_index)); + .collect_one(txout_index) + .data()?; if txin_index == TxInIndex::UNSPENT { return Ok(TxOutspend::UNSPENT); @@ -235,7 +235,7 @@ impl Query { let mut cached_status: Option<(Height, BlockHash, Timestamp)> = None; let mut outspends = Vec::with_capacity(output_count); for i in 0..output_count { - let txin_index = txin_index_reader.get(usize::from(first_txout + Vout::from(i))); + let txin_index = txin_index_reader.get(first_txout + Vout::from(i)); if txin_index == TxInIndex::UNSPENT { outspends.push(TxOutspend::UNSPENT); @@ -249,7 +249,7 @@ impl Query { } let spending_first_txin = first_txin_cursor.get(spending_tx_index.to_usize()).data()?; let vin = Vin::from(usize::from(txin_index) - usize::from(spending_first_txin)); - let spending_txid = txid_reader.get(spending_tx_index.to_usize()); + let spending_txid = txid_reader.get(spending_tx_index); let spending_height: Height = tx_heights.get_shared(spending_tx_index).data()?; let (block_hash, block_time) = if let Some((h, ref bh, bt)) = cached_status @@ -318,10 +318,15 @@ impl Query { return Err(Error::UnknownTxid); } let first_txout_vec = &self.indexer().vecs.transactions.first_txout_index; - let first = first_txout_vec.read_once(tx_index)?; + let first_txout_reader = first_txout_vec.reader(); + let first = first_txout_reader.try_get(tx_index).ok_or(Error::Internal( + "resolve_tx_outputs: first txout index past data", + ))?; let next_tx = tx_index.incremented(); let next = if next_tx < safe.tx_index { - first_txout_vec.read_once(next_tx)? + first_txout_reader.try_get(next_tx).ok_or(Error::Internal( + "resolve_tx_outputs: next first txout index past data", + ))? } else { safe.txout_index }; diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 59b945ec7..a9087d16f 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -16,3 +16,6 @@ brk_types = { workspace = true } byteview = { workspace = true } fjall = { workspace = true } rustc-hash = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/brk_store/src/any.rs b/crates/brk_store/src/any.rs index f1a736869..f024cac4d 100644 --- a/crates/brk_store/src/any.rs +++ b/crates/brk_store/src/any.rs @@ -1,15 +1,8 @@ use brk_error::Result; -use brk_types::{Height, Version}; -use fjall::Keyspace; +use brk_types::Height; pub trait AnyStore: Send + Sync { - fn name(&self) -> &'static str; fn height(&self) -> Option; - fn has(&self, height: Height) -> bool; - fn needs(&self, height: Height) -> bool; - fn version(&self) -> Version; fn export_meta(&mut self, height: Height) -> Result<()>; - fn export_meta_if_needed(&mut self, height: Height) -> Result<()>; - fn keyspace(&self) -> &Keyspace; fn commit(&mut self, height: Height) -> Result<()>; } diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index c4363ed38..f9b4a49cc 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -1,18 +1,6 @@ #![doc = include_str!("../README.md")] -use std::{ - borrow::Cow, - fmt::Debug, - fs, - hash::Hash, - mem, - ops::Range, - path::Path, - sync::{ - Arc, - atomic::{AtomicU64, Ordering::Relaxed}, - }, -}; +use std::{borrow::Cow, cmp::Ordering, fmt::Debug, fs, hash::Hash, mem, ops::Range, path::Path}; use brk_error::Result; use brk_types::{Height, Version}; @@ -32,7 +20,7 @@ pub use kind::*; pub use meta::*; pub use mode::*; -const MAJOR_FJALL_VERSION: Version = Version::new(3); +const MAJOR_FJALL_VERSION: Version = Version::new(4); pub fn open_database(path: &Path) -> fjall::Result { Database::builder(path.join("fjall")) @@ -44,12 +32,10 @@ pub fn open_database(path: &Path) -> fjall::Result { #[derive(Clone)] pub struct Store { meta: StoreMeta, - name: &'static str, keyspace: Keyspace, puts: FxHashMap, dels: FxHashSet, caches: Vec>, - db_reads: Arc, } impl Store @@ -111,12 +97,10 @@ where Ok(Self { meta, - name: Box::leak(Box::new(name.to_string())), keyspace, puts: FxHashMap::default(), dels: FxHashSet::default(), caches, - db_reads: Arc::new(AtomicU64::new(0)), }) } @@ -179,9 +163,7 @@ where } } - self.db_reads.fetch_add(1, Relaxed); - - if let Some(slice) = self.keyspace.get(ByteView::from(key))? { + if let Some(slice) = self.keyspace.get_standard(ByteView::from(key))? { Ok(Some(Cow::Owned(V::from(ByteView::from(slice))))) } else { Ok(None) @@ -240,16 +222,16 @@ where let keyspace = self.keyspace.clone(); Ok(Some(Box::new(move || { - Self::ingest(&keyspace, puts.iter(), dels.iter()) + Self::ingest_owned(&keyspace, puts, dels) }))) } #[inline] pub fn iter(&self) -> impl Iterator { self.keyspace - .iter() - .map(|res| res.into_inner().unwrap()) - .map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v)))) + .iter_standard() + .map(Result::unwrap) + .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))) } #[inline] @@ -259,9 +241,9 @@ where ) -> impl DoubleEndedIterator + '_ { let prefix: ByteView = prefix.into(); self.keyspace - .prefix(&*prefix) - .map(|res| res.into_inner().unwrap()) - .map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v)))) + .prefix_standard(prefix) + .map(Result::unwrap) + .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))) } #[inline] @@ -272,9 +254,9 @@ where let start: ByteView = range.start.into(); let end: ByteView = range.end.into(); self.keyspace - .range(start..end) - .map(|res| res.into_inner().unwrap()) - .map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v)))) + .range_standard(start..end) + .map(Result::unwrap) + .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))) } pub fn approximate_len(&self) -> usize { @@ -286,11 +268,6 @@ where self.meta.has(height) } - #[inline] - pub fn needs(&self, height: Height) -> bool { - self.meta.needs(height) - } - fn export_meta(&mut self, height: Height) -> Result<()> { self.meta.export(height)?; Ok(()) @@ -321,20 +298,69 @@ where items.sort_unstable(); let mut ingestion = keyspace.start_ingestion()?; + // FxHashMap/FxHashSet keep keys unique and disjoint; sorting therefore + // proves the strict ordering required by the prevalidated writer. for item in items { match item { Item::Value { key, value } => { - ingestion.write(ByteView::from(key), ByteView::from(value))?; + ingestion.write_prevalidated(ByteView::from(key), ByteView::from(value))?; } Item::Tomb(key) => { - ingestion.write_weak_tombstone(ByteView::from(key))?; + ingestion.write_prevalidated_weak_tombstone(ByteView::from(key))?; } } } - ingestion.finish()?; + // Store keyspaces are mutated only through these ingestion phases, so + // no journaled Fjall write can race their completion. + ingestion.finish_exclusive()?; Ok(()) } + + fn ingest_owned(keyspace: &Keyspace, puts: FxHashMap, dels: FxHashSet) -> Result<()> { + let mut puts: Vec<_> = puts.into_iter().collect(); + let mut dels: Vec<_> = dels.into_iter().collect(); + + puts.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + dels.sort_unstable(); + + let mut puts = puts.into_iter().peekable(); + let mut dels = dels.into_iter().peekable(); + let mut ingestion = keyspace.start_ingestion()?; + + // The buffers are unique and disjoint, and this merge emits them in + // strict key order, so release builds can skip re-cloning each key. + while puts.peek().is_some() || dels.peek().is_some() { + match (puts.peek(), dels.peek()) { + (Some((put_key, _)), Some(del_key)) => match put_key.cmp(del_key) { + Ordering::Less => { + let (key, value) = puts.next().unwrap(); + ingestion.write_prevalidated(ByteView::from(key), ByteView::from(value))?; + } + Ordering::Greater => { + ingestion.write_prevalidated_weak_tombstone(ByteView::from( + dels.next().unwrap(), + ))?; + } + Ordering::Equal => unreachable!("key is both inserted and deleted"), + }, + (Some(_), None) => { + let (key, value) = puts.next().unwrap(); + ingestion.write_prevalidated(ByteView::from(key), ByteView::from(value))?; + } + (None, Some(_)) => { + ingestion + .write_prevalidated_weak_tombstone(ByteView::from(dels.next().unwrap()))?; + } + (None, None) => break, + } + } + + // Store keyspaces are mutated only through these ingestion phases, so + // no journaled Fjall write can race their completion. + ingestion.finish_exclusive()?; + Ok(()) + } } impl AnyStore for Store @@ -344,38 +370,14 @@ where for<'a> ByteView: From + From + From<&'a K> + From<&'a V>, Self: Send + Sync, { - fn keyspace(&self) -> &Keyspace { - &self.keyspace - } - fn export_meta(&mut self, height: Height) -> Result<()> { self.export_meta(height) } - fn export_meta_if_needed(&mut self, height: Height) -> Result<()> { - self.export_meta_if_needed(height) - } - - fn name(&self) -> &'static str { - self.name - } - fn height(&self) -> Option { self.meta.height() } - fn has(&self, height: Height) -> bool { - self.has(height) - } - - fn needs(&self, height: Height) -> bool { - self.needs(height) - } - - fn version(&self) -> Version { - self.meta.version() - } - fn commit(&mut self, height: Height) -> Result<()> { self.export_meta_if_needed(height)?; @@ -386,9 +388,10 @@ where return Ok(()); } - Self::ingest(&self.keyspace, puts.iter(), dels.iter())?; - - if !self.caches.is_empty() { + if self.caches.is_empty() { + Self::ingest_owned(&self.keyspace, puts, dels)?; + } else { + Self::ingest(&self.keyspace, puts.iter(), dels.iter())?; self.caches.pop(); self.caches.insert(0, puts); } diff --git a/crates/brk_store/src/meta.rs b/crates/brk_store/src/meta.rs index 78fdcb0e4..7c9b24dd1 100644 --- a/crates/brk_store/src/meta.rs +++ b/crates/brk_store/src/meta.rs @@ -12,7 +12,6 @@ use super::Height; #[derive(Debug, Clone)] pub struct StoreMeta { pathbuf: PathBuf, - version: Version, height: Option, } @@ -41,19 +40,14 @@ impl StoreMeta { let slf = Self { pathbuf: path.to_owned(), - version, height: Height::try_from(Self::path_height_(path).as_path()).ok(), }; - slf.version.write(&slf.path_version())?; + version.write(&slf.path_version())?; Ok((slf, partition)) } - pub fn version(&self) -> Version { - self.version - } - pub fn export(&mut self, height: Height) -> io::Result<()> { self.height = Some(height); height.write(&self.path_height()) diff --git a/crates/brk_store/tests/owned_ingest.rs b/crates/brk_store/tests/owned_ingest.rs new file mode 100644 index 000000000..26ad6517d --- /dev/null +++ b/crates/brk_store/tests/owned_ingest.rs @@ -0,0 +1,59 @@ +use brk_store::{Kind, Mode, Store, open_database}; +use brk_types::{AddrIndexTxIndex, Height, TxIndex, TypeIndex, Unit, Version}; +use fjall::PersistMode; + +fn key(address: u32, transaction: u32) -> AddrIndexTxIndex { + AddrIndexTxIndex::from((TypeIndex::new(address), TxIndex::new(transaction))) +} + +#[test] +fn owned_ingest_merges_puts_and_tombstones() -> brk_error::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path(); + + { + let db = open_database(path)?; + let mut store = Store::import( + &db, + path, + "owned_ingest", + Version::ZERO, + Mode::Any, + Kind::Vec, + )?; + + store.insert(key(1, 1), Unit); + store.insert(key(2, 2), Unit); + store.take_pending_ingest(Height::from(0_u32))?.unwrap()()?; + + store.remove(key(1, 1)); + store.remove(key(3, 3)); + store.insert(key(4, 4), Unit); + store.take_pending_ingest(Height::from(1_u32))?.unwrap()()?; + db.persist(PersistMode::SyncData)?; + + assert!(store.get(&key(1, 1))?.is_none()); + assert!(store.get(&key(2, 2))?.is_some()); + assert!(store.get(&key(3, 3))?.is_none()); + assert!(store.get(&key(4, 4))?.is_some()); + } + + { + let db = open_database(path)?; + let store: Store = Store::import( + &db, + path, + "owned_ingest", + Version::ZERO, + Mode::Any, + Kind::Vec, + )?; + + assert!(store.get(&key(1, 1))?.is_none()); + assert!(store.get(&key(2, 2))?.is_some()); + assert!(store.get(&key(3, 3))?.is_none()); + assert!(store.get(&key(4, 4))?.is_some()); + } + + Ok(()) +} diff --git a/crates/brk_types/src/addr_index_outpoint.rs b/crates/brk_types/src/addr_index_outpoint.rs index ca1e8bebe..5494dff66 100644 --- a/crates/brk_types/src/addr_index_outpoint.rs +++ b/crates/brk_types/src/addr_index_outpoint.rs @@ -2,7 +2,6 @@ use std::hash::{Hash, Hasher}; use byteview::ByteView; use serde::Serialize; -use vecdb::Bytes; use crate::{AddrIndexTxIndex, Vout}; @@ -16,6 +15,14 @@ pub struct AddrIndexOutPoint { } impl AddrIndexOutPoint { + #[inline] + pub(crate) fn to_be_bytes(self) -> [u8; 10] { + let mut bytes = [0; 10]; + bytes[..8].copy_from_slice(&self.addr_index_tx_index.to_be_bytes()); + bytes[8..].copy_from_slice(&self.vout.to_be_bytes()); + bytes + } + #[inline] pub fn tx_index(&self) -> TxIndex { self.addr_index_tx_index.tx_index() @@ -28,11 +35,10 @@ impl AddrIndexOutPoint { } impl Hash for AddrIndexOutPoint { + #[inline] fn hash(&self, state: &mut H) { - let mut buf = [0u8; 10]; - buf[0..8].copy_from_slice(&self.addr_index_tx_index.to_bytes()); - buf[8..].copy_from_slice(&self.vout.to_bytes()); - state.write(&buf); + self.addr_index_tx_index.hash(state); + self.vout.hash(state); } } @@ -65,12 +71,27 @@ impl From for ByteView { impl From<&AddrIndexOutPoint> for ByteView { #[inline] fn from(value: &AddrIndexOutPoint) -> Self { - ByteView::from( - [ - &ByteView::from(value.addr_index_tx_index), - value.vout.to_be_bytes().as_slice(), - ] - .concat(), - ) + ByteView::from(value.to_be_bytes()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn byte_encoding_is_stable_and_roundtrips() { + let value = AddrIndexOutPoint::from(( + TypeIndex::new(0x0102_0304), + OutPoint::new(TxIndex::new(0x0506_0708), Vout::from(0x090a_u16)), + )); + let bytes = ByteView::from(value); + + assert_eq!( + &*bytes, + &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "the LSM key encoding is part of the persisted format", + ); + assert_eq!(AddrIndexOutPoint::from(bytes), value); } } diff --git a/crates/brk_types/src/addr_index_tx_index.rs b/crates/brk_types/src/addr_index_tx_index.rs index 61cd16cf4..caea7979a 100644 --- a/crates/brk_types/src/addr_index_tx_index.rs +++ b/crates/brk_types/src/addr_index_tx_index.rs @@ -10,6 +10,11 @@ use super::{TxIndex, TypeIndex}; pub struct AddrIndexTxIndex(u64); impl AddrIndexTxIndex { + #[inline] + pub(crate) fn to_be_bytes(self) -> [u8; 8] { + self.0.to_be_bytes() + } + pub fn addr_index(&self) -> u32 { (self.0 >> 32) as u32 } diff --git a/crates/brk_types/src/pools.rs b/crates/brk_types/src/pools.rs index 5f1ef9fe0..007ef5cfb 100644 --- a/crates/brk_types/src/pools.rs +++ b/crates/brk_types/src/pools.rs @@ -2,10 +2,13 @@ use std::sync::OnceLock; use serde::Deserialize; -use crate::PoolSlug; +use crate::{PoolSlug, Version}; use super::Pool; +/// Increment when pool IDs, payout addresses, or coinbase tags change. +pub const POOL_ATTRIBUTION_VERSION: Version = Version::ONE; + const JSON_DATA: &str = include_str!("../pools-v2.json"); const TESTNET_IDS: &[u16] = &[145, 146, 149, 150, 156, 163]; diff --git a/crates/fjall/.config/nextest.toml b/crates/fjall/.config/nextest.toml new file mode 100644 index 000000000..d7653a80a --- /dev/null +++ b/crates/fjall/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.default] +slow-timeout = { period = "5s", terminate-after = 3 } diff --git a/crates/fjall/.gitignore b/crates/fjall/.gitignore new file mode 100644 index 000000000..ee08ccef7 --- /dev/null +++ b/crates/fjall/.gitignore @@ -0,0 +1,24 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +mutants +mutants.out + +.fjall_data +.data +.test +/old_* + +.directory diff --git a/crates/fjall/.rustfmt.toml b/crates/fjall/.rustfmt.toml new file mode 100644 index 000000000..1c7741475 --- /dev/null +++ b/crates/fjall/.rustfmt.toml @@ -0,0 +1,3 @@ +reorder_imports = true +# group_imports = "StdExternalCrate" +# imports_granularity = "crate" diff --git a/crates/fjall/.vscode/settings.json b/crates/fjall/.vscode/settings.json new file mode 100644 index 000000000..875c86731 --- /dev/null +++ b/crates/fjall/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "rust-analyzer.showUnlinkedFileNotification": false +} \ No newline at end of file diff --git a/crates/fjall/CHANGELOG.md b/crates/fjall/CHANGELOG.md new file mode 100644 index 000000000..cbb553a7c --- /dev/null +++ b/crates/fjall/CHANGELOG.md @@ -0,0 +1,50 @@ +# 3.1.0 + +- [feat] Implemented support for compaction filters (custom logic during compactions) +- [msrv] Reduced MSRV to 1.90 + +# 3.0.0 + +- [feat] Implemented new block format in `lsm-tree` +- [feat] Bookkeep LSM-tree changes (flushes, compactions) in `Version` history +- [feat] Prefix truncation inside data & index blocks +- [feat] Allow unpinning filter blocks +- [feat] Implemented partitioned filters +- [feat] Allow calling bulk ingestion on non-empty keyspaces +- [feat] Introduced level-based configuration policies for most configuration parameters +- [feat] Journal compression for large values +- [feat] Database locking using the new Rust file locking API +- [feat] Rewritten key-value separation to run during compactions, instead of dedicated GC runs +- [feat] Full file checksums to allow fast database corruption checks (in the future) +- [feat] Checksum check on block & blob reads +- [api] Make Ingestion API more flexible +- [feat] Shortening eligible sequence numbers when compacting into the last level to save disk space +- [api] Change constructor to `Database::builder` instead of `Config::new` +- [api] Changed naming of keyspace->database, and partition->keyspace +- [api] Change transaction feature flags to be separate structs, `OptimisticTxDatabase` and `SingleWriterTxDatabase` +- [api] Changed snapshot error type, fixes #156 +- [api] Unified transactions read operations and snapshots with `Readable` trait +- [api] Guard API for iterator values +- [api] Removed old garbage collection APIs +- [api] `metrics` feature flag for cache hit rates etc. (will be exposed in the future) +- [api] Change `bytes` feature flag to `bytes_1` to pin its version +- [api] Make read operations in optimistic write transactions non-mut +- [fix] Consider blob files in FIFO compaction size limit, fixes #133 +- [perf] Use a single hash per key for filters, instead of two +- [perf] Improve leveled compaction scoring +- [perf] Improve leveled compaction picking to use less hashing and heap allocations +- [perf] Use `quick-cache` for file descriptor caching +- [perf] Promote levels immediately to L6 to get rid of tombstones easily +- [perf] Rewritten maintenance task bookkeeping, and write stalling mechanisms to be less aggressive +- [perf] Allow `lsm-tree` flushes to merge multiple sealed memtables into L0, if necessary +- [perf] Skip heap allocation in blob memtable inserts +- [perf] Skip compression when rewriting compressed blob files +- [msrv] Increased MSRV to **1.91** +- [misc] Blob file descriptor caching +- [misc] Use Rust native `path::absolute`, removing `path-absolutize` dependency +- [misc] Remove `std-semaphore` dependency +- [misc] Remove `miniz` (will be replaced in the future) +- [misc] Use `byteorder-lite` as drop-in replacement for `byteorder` +- [refactor] Changed background workers to be a single thread pool +- [internal] Store keyspace configurations in a meta keyspace, instead of individual binary config files +- [internal] Use `sfa` for most file scaffolding in `lsm-tree` diff --git a/crates/fjall/CONTRIBUTING.md b/crates/fjall/CONTRIBUTING.md new file mode 100644 index 000000000..b2bba5915 --- /dev/null +++ b/crates/fjall/CONTRIBUTING.md @@ -0,0 +1,11 @@ +# Contributing + +## License + +By contributing to this project, you agree that your contributions will be licensed under the project's license (MIT OR Apache-2.0). + +Thank you for your contribution! + +## Looking for issues? + +https://github.com/fjall-rs/fjall/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 diff --git a/crates/fjall/Cargo.toml b/crates/fjall/Cargo.toml new file mode 100644 index 000000000..1c445600c --- /dev/null +++ b/crates/fjall/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "fjall" +description = "Log-structured, embeddable key-value storage engine" +license.workspace = true +version.workspace = true +edition.workspace = true +readme.workspace = true +include = ["src/**/*", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] +repository.workspace = true +homepage.workspace = true +keywords = ["database", "key-value", "lsm", "rocksdb", "leveldb"] +categories = ["data-structures", "database-implementations", "algorithms"] + +[lib] +name = "fjall" +path = "src/lib.rs" + +[features] +default = ["lz4"] +lz4 = ["lsm-tree/lz4", "dep:lz4_flex"] +bytes_1 = ["lsm-tree/bytes_1"] +metrics = ["lsm-tree/metrics"] +__internal_whitebox = [] + +[dependencies] +byteorder = { package = "byteorder-lite", version = "0.1.0" } +byteview = { workspace = true } +lsm-tree = { workspace = true, default-features = false, features = [] } +log = { workspace = true } +tempfile = { workspace = true } +dashmap = "6.1.0" +xxhash-rust = { version = "0.8.15", features = ["xxh3"] } +lz4_flex = { workspace = true, features = ["checked-decode"], optional = true } +flume = { version = "0.12.0", default-features = false } + +[dev-dependencies] +nanoid = "0.4.0" +test-log = "0.2.18" +rand = "0.10.0" + +[package.metadata.cargo-all-features] +denylist = ["__internal_whitebox"] diff --git a/crates/fjall/LICENSE-APACHE b/crates/fjall/LICENSE-APACHE new file mode 100644 index 000000000..0e5254213 --- /dev/null +++ b/crates/fjall/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/crates/fjall/LICENSE-MIT b/crates/fjall/LICENSE-MIT new file mode 100644 index 000000000..97f5e0faa --- /dev/null +++ b/crates/fjall/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 fjall-rs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/fjall/README.md b/crates/fjall/README.md new file mode 100644 index 000000000..0dbeeaa15 --- /dev/null +++ b/crates/fjall/README.md @@ -0,0 +1,207 @@ +

+ +

+ +

+ + CI + + + docs.rs + + + Crates.io + + MSRV + + dependency status + +

+ +

+ + Discord + + + Bluesky + +

+ +*Fjall* _(Nordic: "Mountain")_ is a log-structured, embeddable key-value storage engine written in Rust. +It features: + +- A thread-safe BTreeMap-like API +- 100% safe & stable Rust +- LSM-tree-based storage similar to `RocksDB` +- Range & prefix searching with forward and reverse iteration +- Multiple keyspaces (a.k.a. column families) with cross-keyspace atomic semantics +- Built-in compression (default = `LZ4`) +- Serializable transactions (optional) +- Key-value separation for large blob use cases (optional) +- Custom compaction filters to run custom logic during compactions (optional) +- Automatic background maintenance + +It is not: + +- A standalone database server +- A relational or wide-column database: it has no built-in notion of columns or query language + +## Sponsors + + + + + + Orbitinghail + + + +## Basic usage + +```bash +cargo add fjall +``` + +```rust +use fjall::{Database, KeyspaceCreateOptions, PersistMode}; + +fn main() -> fjall::Result<()> { + // A database may contain multiple keyspaces + // You should probably only use a single database for your application + let db = Database::builder(path).open()?; + // TxDatabase::builder for transactional semantics + + // Each keyspace is its own physical LSM-tree, and thus isolated from other keyspaces + let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + + // Write some data + items.insert("a", "hello")?; + + // And retrieve it + let bytes = items.get("a")?; + + // Or remove it again + items.remove("a")?; + + // Search by prefix + for kv in items.prefix("prefix") { + // ... + } + + // Search by range + for kv in items.range("a"..="z") { + // ... + } + + // Iterators implement DoubleEndedIterator, so you can search backwards, too! + for kv in items.prefix("prefix").rev() { + // ... + } + + // Sync the journal to disk to make sure data is definitely durable + // When the database is dropped, it will try to persist with `PersistMode::SyncAll` automatically + db.persist(PersistMode::SyncAll) +} +``` + +> [!TIP] +> Like any typical key-value store, keys are stored in lexicographic order. +> If you are storing integer keys (e.g. timeseries data), you should use the big endian form to have predictable ordering. + +## Durability + +To support different kinds of workloads, Fjall is agnostic about the type of durability +your application needs. +After writing data (`insert`, `remove` or committing a write batch/transaction), you can choose to call [`Database::persist`](https://docs.rs/fjall/latest/fjall/struct.Database.html#method.persist) which takes a [`PersistMode`](https://docs.rs/fjall/latest/fjall/enum.PersistMode.html) parameter. +By default, any operation will flush to OS buffers, but **not** to disk. +This matches RocksDB's default durability. +Also, when dropped, the database will try to persist the journal *to disk* synchronously. + +## Multithreading, Async and Multiprocess + +> [!WARNING] +> A single database may **not** be loaded in parallel from separate *processes*. + +Fjall is internally synchronized for multi-*threaded* access, so you can clone around the `Database` and `Keyspace`s as needed, without needing to lock yourself. + +For an async example, see the [`tokio`](https://github.com/fjall-rs/fjall/tree/main/examples/tokio) example. + +## Memory usage + +Generally, memory for loaded data, indexes etc. is managed on a per-block basis, and capped by the block cache capacity. +Note that this also applies to returned values: When you hold a `Slice`, it keeps the backing buffer alive (which may be a block). +If you know that you are going to keep a value around for a long time, you may want to copy it out into a new `Vec`, `Box<[u8]>`, `Arc<[u8]>` or new `Slice` (using `Slice::new`). + +> [!NOTE] +> It is recommended to configure the block cache capacity to be ~20-25% of the available memory - or more **if** the data set fits _fully_ into memory. + +Additionally, orthogonally to the block cache, each `Keyspace` has its own write buffer (["Memtable"](https://docs.rs/fjall/latest/fjall/struct.KeyspaceCreateOptions.html#method.max_memtable_size)) which is the unit of data flushed back into the "proper" index structure. + +## Error handling + +Fjall returns an [error enum](https://docs.rs/fjall/latest/fjall/enum.Error.html), however these variants are mostly used for debugging and tracing purposes, so your application is not expected to handle specific errors. + +It's best to let the application crash and restart, which is the [safest way to recover from transient I/O errors](https://ramalagappan.github.io/pdfs/papers/cuttlefs.pdf). + +## Transactional modes + +The backing store (`lsm-tree`) is a MVCC key-value store, allowing repeatable snapshot reads. +However this isolation level can not do read-modify-write operations without the chance of lost updates. +Also, `WriteBatch` does not allow reading the intermediary state back as you would expect from a proper transaction. +For that reason, if you need transactional semantics, you need to use one of the transactional database implementation (`OptimisticTxDatabase` or `SingleWriterTxDatabase`). + +TL;DR: Fjall supports both transactional and non-transactional workloads. +Chances are you want to use a transactional database, unless you know your workload does not need serializable transaction semantics. + +### Single writer + +Opens a transactional database for single-writer (serialized) transactions. +Single writer means only a single **write** transaction can run at a time. +This is trivially serializable because it _literally_ serializes write transactions. + +### Optimistic + +Opens a transactional database for multi-writer, serializable transactions. +Conflict checking is done using optimistic concurrency control, meaning transactions can conflict and may have to be rerun. + +## Feature flags + +### lz4 + +Allows using `LZ4` compression, powered by [`lz4_flex`](https://github.com/PSeitz/lz4_flex). + +*Enabled by default.* + +### bytes_1 + +Uses [`bytes`](https://github.com/tokio-rs/bytes) 1.x as the underlying `Slice` type. +Otherwise, [`byteview`](https://github.com/fjall-rs/byteview) is used instead. + +*Disabled by default.* + +## Stable disk format + +Future breaking changes will result in a major version bump and a migration path. + +For the underlying LSM-tree implementation, see: . + +## Examples + +[See here](https://github.com/fjall-rs/fjall/tree/main/examples) for practical examples. + +## Contributing + +How can you help? + +- [Ask a question](https://github.com/fjall-rs/fjall/discussions/new?category=q-a) + - or join the Discord server: [https://discord.com/invite/HvYGp4NFFk](https://discord.com/invite/HvYGp4NFFk) +- [Post benchmarks and things you created](https://github.com/fjall-rs/fjall/discussions/new?category=show-and-tell) +- [Open a PR](https://github.com/fjall-rs/fjall/compare), + - [See open issues to pick up here](https://github.com/search?q=org%3Afjall-rs+label%3A%22help+wanted%22+state%3Aopen+&type=issues) +- [Open an issue](https://github.com/fjall-rs/fjall/issues/new) (bug report, weirdness) + +## License + +All source code is licensed under MIT OR Apache-2.0. + +All contributions are to be licensed as MIT OR Apache-2.0. diff --git a/crates/fjall/commit.nu b/crates/fjall/commit.nu new file mode 100644 index 000000000..a0a3001bd --- /dev/null +++ b/crates/fjall/commit.nu @@ -0,0 +1,64 @@ +let machines = [ + # Fly.io performance + # "fly.performance.1x", + # "fly.performance.2x", + "fly.performance.4x", + # "fly.performance.8x", + # "fly.performance.16x", + + # EC2 T2 + # "aws.ec2.t2.nano", + # "aws.ec2.t2.micro", + # "aws.ec2.t2.small", + # "aws.ec2.t2.medium", + # "aws.ec2.t2.large", + # "aws.ec2.t2.xlarge", + # "aws.ec2.t2.2xlarge", + + # EC2 T3 + # "aws.ec2.t3.nano", + # "aws.ec2.t3.micro", + # "aws.ec2.t3.small", + "aws.ec2.t3.medium", + # "aws.ec2.t3.large", + # "aws.ec2.t3.xlarge", + # "aws.ec2.t3.2xlarge", + + # EC2 T3a + # "aws.ec2.t3a.nano", + # "aws.ec2.t3a.micro", + # "aws.ec2.t3a.small", + # "aws.ec2.t3a.medium", + # "aws.ec2.t3a.large", + # "aws.ec2.t3a.xlarge", + # "aws.ec2.t3a.2xlarge", + + # EC2 T4g + # "aws.ec2.t4g.nano", + # "aws.ec2.t4g.micro", + # "aws.ec2.t4g.small", + # "aws.ec2.t4g.medium", + # "aws.ec2.t4g.large", + # "aws.ec2.t4g.xlarge", + # "aws.ec2.t4g.2xlarge", + + # EC2 M4 + # "aws.ec2.m4.large", +] + +let table = $env.TABLE_NAME +let commit = $env.COMMIT + +print $"Queuing ($commit)" + +for machine in $machines { + let q_pk = $"q#($machine)" + + print $"Adding queue item for ($machine)" + let item = { + pk: { S: $q_pk }, + sk: { S: $commit }, + version: { S: "2" }, + } + aws dynamodb put-item --table-name $table --item ($item | to json) +} diff --git a/crates/fjall/compile_examples.mjs b/crates/fjall/compile_examples.mjs new file mode 100644 index 000000000..5812a12bc --- /dev/null +++ b/crates/fjall/compile_examples.mjs @@ -0,0 +1,60 @@ +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { resolve } from "node:path"; + +const examplesFolder = "examples"; + +for (const exampleName of await readdir(examplesFolder)) { + const folder = resolve(examplesFolder, exampleName); + + { + console.error(`Testing ${exampleName}`); + + const proc = spawn("cargo test", { + cwd: folder, + shell: true, + }); + + proc.stdout.on("data", buf => console.log(String(buf))); + proc.stderr.on("data", buf => console.error(String(buf))); + + await new Promise((resolve, _) => { + proc.on("exit", () => { + if (proc.exitCode > 0) { + console.error(`${exampleName} FAILED`); + process.exit(1); + } + else { + resolve(); + } + }) + }); + } + + if (existsSync(resolve(folder, ".run"))) { + console.error(`Running ${exampleName}`); + + const proc = spawn("cargo run", { + cwd: folder, + shell: true, + }); + + proc.stdout.on("data", buf => console.log(String(buf))); + proc.stderr.on("data", buf => console.error(String(buf))); + + await new Promise((resolve, _) => { + proc.on("exit", () => { + if (proc.exitCode > 0) { + console.error(`${exampleName} FAILED`); + process.exit(1); + } + else { + resolve(); + } + }) + }); + } + + console.error(`${exampleName} OK`); +} diff --git a/crates/fjall/kawaii.png b/crates/fjall/kawaii.png new file mode 100644 index 0000000000000000000000000000000000000000..7e6152a5df4d62988b2fea905b6006d2b42980b3 GIT binary patch literal 92194 zcmX_oby!v16E2<7-5x-?MLOir-O?R`bT>$MNq3hZEh(MSp>SxFmP1G!ghSlT_q+H0 z;o~E^_gZV#%scPQyxUj}HF+E?N-P8f1RO;L87%|^6d?ozBufl5;3xDy-5G)35IwZy z!H6s0IYNLxV7e+8cpxCW`SJV<(bQ%S0|D{ILKmvBJjZ&_M)Th$Pp*(^w=Y@R}>U+X<`}AZZe~xsKt~}gv5K}R!my_m$aqHpR+Wu?v{Y=m-bP6XM3VQB zEF_IIhRGv`N^`t|y~^h)T>-IWs_=auwwA0X1V2BmMWM0%vV5?q#*c6MKN)%Ys5jx~v6H2X) z)(FNFifTU~Z=v$8Jv_gA1MBd0#^>siW*fm*8cosxp(@A$L4Y}cRTt=4T=BIKQTASS z$2+}XHmxpBB@+6vdV87lLn`v~#gDR8*Csm-Z)V3$# z4QBPmH!gZnz^AqMlXc-8f0}evR{}|ZA=DXm{qou9AZt`&Aq-Z{FfF3#VST+iJY(-D zTYJXv*~_!;Z%wW_v_+QvRLg>-f&OgS2Q$dlnNkvOxMj3t|EmWc-byt(HVAD(fiP_#>kA1*emr88q4Q z%uNaVKNn!t0}f0Ga%9_}aL)a{YPlBU#M;)QuU-_K+Wa;$amA7jO#sQ+@x(e!MF;Mg z@Sl}Zz;8pt@n<1AT~x*^-$^au7Lon)BUNVQPy-BR*95WvT_{6`dnT%B5B+}!F3$iq zGK?&cvDDXdJ<>)BkA-{)#rbHc0HWHv*%4(~eTTfM=iXl?hRhT{K*5oEoiQ`A@ZoV=OOVMi|`i1UhP-yQUe#Egu59wrPO8U8R#8vhfGjyueuKA+&2YpHj}YG+_f#Quxb zxmVDAvz+lbjWV5R07-otHtjM^qv2ZaU$E^PYrJ}=eGekA!tV0{2{Bif zqS>WaD2d--#1c|^CP%QF-Hx`uj`$%zL+5W%PZ6Tn^mCib^mX93ETuS!MpZ8mni@d_T0hf0)B++tfLc(}wg${sb0%9SBk(W|6wsww;Z4D5Y zx%+Oqh?L)DKC;}TRIdaK!=Q?Q^ML5#9;cRO%BWXhv$K!2|LVvd{o4)C_-a#%b5Odn z@Lu%R`?Or2`j~Yje7uAEt9c&Miq&&Y5#@o}Y+a4P6BHZax=>#!s3I>(e&45_j@4Q{ z00dEhwO1HIFU<{-2t~y3Q+XbHf%v1kzNkzT?8B`659p83uzmN=~VN(%LWb2w_i z2F(A;wOU`wA{5wTj@ROd1b%=Xlwc+PJ@D`F!52Uzq;nF!b>R8d38}FZ9YUQ0@{Wx5 zesuf>6O8waK9jbY*};1Rq3K~bn0|@Q*=(sUfhFTT@MM{|Ol52jvv&b>sK^%- z&?lXQ@a^%2cn@|5m+`}4X0SMYM8wC;FZrwjxXws#S6d_*@j}Pz3_L{RPF}W2W`{~C zbQZ(wPY0G`AM{|qYM#R`h2!~@F9%vTDa6p#e(1|PHku_X#LsP1Pjs-|>YrnVlaO3{ zN$TBPdPy#Yp&6a+K%c(ApOlg@C8|yqKR90AJ*BafRIqIgR;ZY94_tH1KBge7#DAq5 zO|_IHU3=$9`~}ZG;)B_!0sq{^++RLp@Wx-K8QcUm|0Eq5 z?LGRX$^54Yv#I>IaoonVkX5Ri+jXmtB$jfxR__8Owmwi99aodJZ7%EXR%hU4qDojd zf>CO|vP;aM&~Y~~WTvnbdu`9gi?hQ%p)ALX!?owfgX{K~CCS!k>!~9vm?L!1tVU=_ z;qA~Ns~}Nnj2dGW*OX6;)qAGWj#N#OVP()bd9~tkOoGm-c<%v9?BRHs^Cx#kCzE}7 zwSO9+a%o-;Cs>HDv=5%j^$$&k*|$fh9$!icR8-Tygg2@2uP8$UJ4Q<)QYK`nkQ|J? z`=A4t|1BdnarGxWU6fG61;-~#qeWop9IAHvX}Wpb#=IYs$`JgC$9YtWLw}9plxfAC z4Ex7mEN$0*M2az!0@FWA*<_wuDX4uDI%5az(OQnX9L!>RMxU;pde5#2^nB|wC{9>G zv5-sXv=j!bKt`}4Sm_03_4VmCt0g?{3#N~sDweF+`0#i=6Zw+9`u2AHJfC~1oZ9wy z=1_k|*WuV?_-rb;w0QxgDQoxjYv7 z=+iSXo$f@0;`~qM_M>kkerjdqzFZ97<8=D9723lY3ZvheNvD@;z;W6Q%z=C;+yF(S zqxqAAm%=m%InTV7@9(1m>9fGYSeDDO3 zDK5d80>XxBrt}Cdu_yxcMJip8uE=P9<9x5$5$02zqKVF8HRgm~CaUGwrx%k3l=5aE zP9r}#C?;kXnJZ;KIbRCkPJ4U}YbTrQw91u>ALKJX1=71_gvK;R@VgRBt-E z^wIlhWnYl+tfJd6pCl3qyIUDy+Q+Ty8Ra>k2F9Kso8B9jhkz2<_(|kJQbQXY7-lmJa_JcIgF~s3~j}^S?X70|0J5 zb=%+!<53ejW?3NCy+ilFu45rRsosSWCPxCdPf4jUnkZeNi8mP4ivG~i9sk4}db8GB zz{=Dt>B14OmaTaAd|j~ek<*kQnCE{+KsN=amV!cC;wYkH3SPBQu!##wJ>9kP+L7%{ zXngKJVjjkuJidbJ6Z5mbx{I?D7{jqtS48NAv`CgP1YA+NiVJ$*0MSrxnT@FT`6Bl} z4?t|FR5(ZKa2|o=#WA6|gUD{`bCJiKZ5t zkb>jqsI?!eK{S5_=KpfH}e&iHxrK{i!Iy{DTDxXWfdaia5#7{UXqia zJtZca{$Yo{tX<~8g9C6Y>5-(!rEBQid`m{wN)w9nw`&u^nAyt{6^GOwLv%GdWS@g{ zd~je{1>5p?uYb+6K`XEudKIKnKlrA_Y4RZeHhXD{?Wz05NVmG@U3Lh%rU{i`26yH&08*>^zL8r^K$)PVE@FT==x8x$lDzRBS>b8@^5X|WJSk#}?Z8ZS zs!2=R+j^RVS#(nFOzeXS_g~*MxJt7I{ybg(y%T=u=voa@JzCY}`nkK@q|NMSCW2tn z)qgm4vWC3T)v5cm)zb|RDU#p`f+Kyn;Pt;cha09Nv$;>zcscCxAUfwmoe0ypFQLjU zDM3az(7roZIKVc}Lm>psMQT?k7@+d2?FlECj7C7u{cn)*@A zd^rh3v^v?@uhFylP`wuz&G%K9Wu^*_dzt8r9`q%v1|#Cg?|L*}()tWWW4$U68MuNT zJvNla1lRpm>Dfl9&bsUvqA=gr_p;EdJpyERjX zwbBhA=hJfJ-0H;#E6{pU-xydXXYSZJgB3c?u@YOcFjMv>aG3{G6%58&M1Pg-F`Mim zwUxOg%CJ9hGmp29W07wK9mpiqXovb;n(Gda=U|cN+G}!=?=-)4x0GM5=V5cT+aK!> ziA&Is;|0QQ@cuitocGuIAVF^G!%P^A!ix{zwv7}K3Ky8V;GVv@87sBkKleN^l z@y(ug4!t<1(={?MeYo?a$fyZ(vF^ZL*=T~%&H(b2adpVbBvMmb<;>0TRjYTW+xdWM z%KFV{_~;HBOu&1U-;1`J4J4Equt~aUH_!Op@FXLmSlIJB#*Cnh8{Ml2nLMndRz>C1 zetq>{t8aB2rUfsdYQA>o#WpPBY~_7XXii71lY`j;IPYHUj3L*ge>-5!Dk)hZ*JNd2 z)oqQCZ4u`mV#q;RGCe=2j%Axnef3aR;_sSjJP?!3onoIl z&)yK+c3y``(%w6Af|%b~JK777o00NIn^MQn%zt}dviMnGTZROzpv*Erlk@K-IkRsu z1l97`-M#ayEcz(3d(DLH;pdWsQOvF6+A@=*#QT@Qd#a^#?-HX+)9u*u9 z@FBa{$?buD*xJzjp?rOxyl#dI-jH-;V#Pqg@U6`k_rvj_b?mpj09)vQP%fFkbv|rH&>K>vt(bm$uTi&;XEqED`jw@!9Y6}Ye`?L98ha=Yg+to5HRl)AqAIkL#Hu+4B zROK%XChtbk={g*7NcdZmqtxgvqN2?Bhos^DFKD67B0A>~Z=>Zo&qIx3EBP(jsBqJ_ zoIkd^1U~!Vuf;uDe5a!+OnLc$Cqa`MbU)jCPoODzqlaqMUfW3T@OEBLE5wY@%OCW` zIuV09k`(_#lRFwRXs`dmoV$>$Gw~l+(`%cjI@mKX@W*(kNPi~L2vCbq?f(HN5f4qUK;hXt_|Wkp?!K-ciPPCClFa-l2{Y|{STM-d+XRw9Am ze<0D%B$M>QN^uM4&b8M%XA~ouGq}}_tbYF{GrL9MrVe&5Z@$J^2 zjfb)R6!K(na2!WF$z`@`>3b5AH||#LlgAv=3P+_#Ew=85v`{n0rI-5zO=;F!yC#%i zg|zhV88obngc!#(z0$)v+bj*jStns`4F!|d>kg_o(DBs%EHhi?JZt8ByA};6tPf>w z$B_{cw$65FV}ELg>N;(`EwQ6;!dLu{7tS_Pskm!LznfuBmKD;$6%9L%XsD>39MTxK!R5FG()B{AM^Q<-xuAIStN7YoZ!pVn+C+0%V<) z_2k}P`z`;+Q&MZE^gkwG5{~^D)JnC;pljXj0-yY1-uFo^vkK{lKe27|oIWNBMy*)k zWu45I$sc7l+c|!jJ|nfAP(7RzO$FKrreU!5K{1V#o|3Q$Lkrp}gTn@k_bU+rV@48R z!|=6?J!cEvQAPIn)IjW;^)^qQslyfPd`uQGCFV!3d98G#5tvNpz4NYIuQ%>+qLxe~ z&h*4Tn34WT707lgkf{2)NKlDhMGUuPlB++{XfVd2EYX-kxdw}-^HYY6%^SmaA)oYy z)yR|;&rQ#f$2aRY?7#Bwxi+bxO1@PdKN(|l=5Yv5S$O+eXn#r0Vk<}=HeU9ecVi_I zcoeu0meNMuEJgjit6c5wJgSP_Ojtiqu{kD?Bf=-x8|gw`1m4i_1PCarD@UAp;YO}FG&q(md?FUnSzK&tj(6(3KA3xv6fi>j2_aY!RHP5Pnf@g=#DsnO?F*PH z29}t0R?Y^VAlUeDfbt-8@7!%Tla(&+-loTm0ZNV5l#65-IQ!6SjPB#paHXu}zVWXm zNI*&DXLXGM@91@+VDq4yHzgJ*vhuw;vnTXh3zc;D09wUzdzlD}k-<9wBE8IbaVGCx zlyhyY>+4J0 z4>x7cv11jw@MNMPE)93|#1y&6!cxvcTC5xhzI_Gld%_8XQAFZsQMW1{Rpt#+2XG{0 zHeQ{bRko&M9lcDm3cV4Hj|4Ba)RT0}h5hSv3g*iysCvaC>3r<*6>}PLNT8w&(0it? zEw!4zJAfF$ z0q;lVa*%dXlxkVTw5Kf)K|HQQZaMAZ8KX`s7!8{#2nzTqrbDPtsA6QGF8~lOa978d z8c6eO5v5p`B!8`1fOGnXnwLP6`SpjOEc8FwC$%;%-xjNgB2w(M*|NgQ`xUm~`5bUK ztnB5BYWE%irN~u7i0J9ZhXcvbqs0~Na2f2A>G?j%sqf4~`M#EI2KL{JdxR^0{)3fj zh;d8ZBYt_XUD@fL@H%06+Br>YQYQd z7`K0tKIDE542O&sZrD>%a{FNA#>n+7%P}FV;vSwM4A2X^_4Mddg!u=h&5(7AaE#?f`tw%D z5ou!p(5PN;)sHu3B*nH9Sw-c9F zM~%r6Snvhq72XE@VJh5ml%oaIxr+>B+>gRn8pSSVTeM#&6MM4$*bNn9+S{iQc56Ui zVIGq4A5dBM(a}#bSgu7J>8J4ogr{6cw+K}RoQ#xMo+&odZ5~V`gP*4YnjnRKxS&BR zZFhtS#**|%!s&(jEFngA7RUF@V#t~^GAY;92@WW-OdcmF^tW9MTMO;Erhzejj? zV#OB^4x(-_(j9y5rTNNPa;U@Tfs1Xdfd4zUGUTBb;0Y{F^?6h3;1MgDnxIy7^ylNI z%I6UMuy~@Ze;s09+E0 zBNtAxW}3i&A3}!BeB2o0%0`Dm+E6Y`Xf0L1N9e7IwV#0_ywoov*;M4sE7}JRQLw^f zPMTZ5XGzBGLDFWj;JxPy=Z`)ANpH_ic=&;J;^?_N`m<87RPvoNx2YsZpftCm$rS`; zE<#9_#D(wYx&JO28Z9a3n~N&%@gIOTW4l5uc=DW>L{(+lDHBJ&N=DmyraQaf($EC- zK*R|l>Jv8zD>q;Lt`^Tbt^TCK^>_iJyW80F@GWtxPfGYUx;)=@_`Ey?L+Gg!z#r;> z8pbeEpmOnV!v_7TgS2eIS#S8mn522%KO}NdQvJ56cD)GVBG^gj^-}nXuT{3T8sd`a zS}dKWw8JQw^}3KjpEkm37#3Bsl6i>p1NV4gLrYK*$=cUL)1R`hPw&P;KVv-N;)7R{ zPAk$fk%QkP-C7P&5(Qv_GO){%`DDV@OPcM@13o?PB96jjx5$)0)nzFmaoH?7=v?aU zYr)S{)1=Jh7l$0nkUS?ekE?l}?i14K1YDe%gC9uod=gZ9Uo%4DOt=Rg$X70xKP9&l zi)uQ$^f?b)Zyu74Oy}Wi@wCd=)ZaYHh2VF~!3I9*)?tDoqmkru2x6$(PbB*W_8r zzmew@qLrkDnLUNka2+hLYVSrA!cGW4^USFTXgFajK|a|#$>>XllkM{CWj#WzIyP-L zu?SyeWs~eJ=lv{1CmzM;w$4ex3aQmi&fuR&4fd{tDZen%RK6WQ`&Vmmza~?a00x=45aZ1>)XbZF8m3G01bTQkr9*bbuYxC}tQr)gk4sSn)c5QX1RwVwR z3}54Iep#-iVd3llg-G>x-SM!G8t{(|y1DLPl zKPjnVV3LqTzaNk49IoyEn+jQC%3;|^Qi@0kBP^*^S};^<;{e7Y&ICk)l`5e@e`O(K zbm$UK>(BdV7O{|JuBQ(66{wY-)Tfu`N*i+NH}$WspKoU~gbRZh1NYH(+8;_jU4fW# zd}p4o;jmw!JOXn<>y&J5qMe(Q9@OV89{wqvAVqdwOa|Co{Va?2|qI{u7GUpQp7v%2LN z@20KI8@}W)A#0B8Ip<84hiZc0aJ`b%HRBe zpW>n>5tcAcYPAwqILx^W-MFmoes5ehL5FWWO(^>IzJHGang+KBwqhQc1`Onf^gh|M zckY+ve*bl&X2ZX|OAJ=P<=n|LJ??&=isJV#T#-ovc7y^N=1x#jiulB?96Mnm&MEJA zXAfza%}jGnbaBA|eX012wI4-gjw{AP`JC@N9;q`DPA?xI0pa*ag0ePpjIfc{s(fx~ z&p`WF+)v>4$J#kv&LD1ceo?M8_bb88p}Z64`6yN6tT;M+>UDp!4Q&B1^^1U=v>-(b zZzGtuZ`@`}%fBfrBUFCZ+GKYo6ySmne%Gh~8|tJG6j`Ec$hNlRE&^cnesG*| z5V$cK181@|+zO4%(q+n^-;c^#q#Pf6ZcO!Q9z;AU<>fIzC3EfEQw?Ywu-rtz7Iqum^x_z*6R^Cv4qi%s6D;U`-#+5;&` za{wQ337^nO{F%9KOuBBE*2ncL{}O72YHY`l1G8w>$v2kKvDa5nwZ+5qbXv<3u}8K< z^3bX4W#pouOd3>Wj2fOUZ`9x`hT$1ubciigNU+^TXJ&2sa74yFbj2??DI(D9sbfpL z2w=!MP?PO96;5dR{k-b{zTivpfMbD0BnY@X%935oa?7>ntY_XAi+XeL^}%V*zcahL zHLVxXMd0_fI-onYQCKdUV%K%oOGBchzyL=6Uu`;-9p68%LBj8kmO~{G?2*Y88FcRkS#v?}h0lZo!kbT0vWZ^H zUOU!a3rcX<$c+>-{FZQgwl)zsrc$VMmGyNz**X~!y3=g5r>DG z^t_0XjquNQy4rH3VBM~fBf*h48_;7)X^QtcSD=HJ%C?Iq2iK=d{bA` zF?3tib5c;&7U8je8PSSBBDC+X=i_71q6uchmSd*{oR-Xhi$HX?ox4`XW%c@tyQ*xw zD$~xLlW~YR6q}p)>0d-jb3YWQ4_rd9U(jF;i(!2A7xb`IMUhD&J>9uGm$>lvfPwF=I78C?qlLXzZY0svZP}*`qN9g=)|Wi> zX4x9<4754(Aa)GH{ApJ3lfAGWkH5(cs`A%Km3paVkySw(cU~ME&!}NIR8INwpH2V71!;)!XjTM>cT(4I*`RT1tB@)Q#B zxCuh+sdIz+g|MKW_N`52pyMx}moSq#?Oe0Wk@VM6639~uMQXsUBBLhJTWKGi{2CpA zU7RSFF@{BzN=#@K2H@s&pP|VmTE5lTq2E*k3LLBw2dEG@!|?;iMa=($22G^FKS5o0a$D z`2k!!&eS6~V1<7=x;FCPN9+4&A=YWXX^nsizVhfB!YrfYX@wJFqT!}IBlp0=5Qdm~SdI1yO;J}QEscb~}P2Z5j4K+`Iw_CC* z2B;_CeT&f4%KJ%LNJGaM*N09e>=B%kKAqh^G++ga(y)44SB?t_K5uz|^X;qwoUa4e zHK9PpV>E{(+2veUAH_d5<(m2^@kJWG5+1)@l{=7NBkMHpzMWJu8RTOgTc!0Od$=wYbtkab)|_Iom-0SyeU&*laE; ze4L>%m$ui_o&Zk;1LSu&hq#~bVV)#Lc(_>QJI1L5>{ji@Fte0`IsqR86bZiRJj?1X zq<$pl@@@lC?DZd15Tg!xZ$yl0g)SFF8>((82?H$Qd1bccJ3pxfJ^$61WwL+gWc{s* zgAg3Uz_bRNE8)p9Mrzqf8WC^HtevXEv50WQhTY&^KtkQ*(1FQj8GH@gz+1Gb3QvY8 zGUez)7lBriFHZy}0ieH0-}7z~9~SZVt^(zD&c8nWQx_kZ(I0eWXU)||?R&HFn&PpO-*!rT4 zH*tJeJoU8LPoLP%O{xfdLbORv9_}5{cxkHM0^3iq-GL%f7?^li)Xzdk?`CQy7*Jwg zl;%(c)y4&6Ip6y(yX?$}hK@{O0r80ej7;)<>8A|dBhtiY$F%ED>SFPlR8YXbNt8=# z)a(CT+93dsv*}kdSPij^I1sqxSz9tcW8+{rk~|>k^2auKrhiZ(j8t2 znamt8^}vA+;UQlhkezh^c!bgA6Ws~r%SOhVQZr(u%pECN)Lr##X`!g{E67~fAQmi3 z#@EWJwyP~`%JL-A5Z=5Ii{R_p-!|@c_eDg)hF+X=Q;EHLQ)AS+sh z&#m>DTt!CzQVDIUxiYhk-A`~TNMC4}$fza`@APOf0IHDCy~%h_4KM~~toMoe;}vuE z5FUD$7yAtFnrC}U&K|47J6=j4)8TQd-2Vul1kB4es*As3ZdS-22HM*vjD$SsG$@w`l9Os?g}BU=x0eg`i=$#4|L*TfoQa;S)y z`Zqn29awVziedxuK`b?neahs*Ac46udAFpRUpGG}Uv8|cB-!R~K1Zwo>8l9ew>Pb^ zlR?P|L3Ga6P{syR;3+KBBqYFTer^vZ(!pUBi2hdfAsOuItY~S>3?8fY{ql6*v3I_P z5`!WlqIr3k^AS8QzV{bPjR;q&y#q9HwkG)eO$Y6Q^*3@&GdRG#7acVh=>{Z{)c!(A zB7440R*nQmtdFV{A?%YGF^1Zj)9Z7iZYJ^R!D3F7d?HGJ3w+`{#J~Vw#}$3)eL4Mh zt9L|rvoyg$q{~s440T%Q>ka4c&QM^0s7Vvk7JL>h&f(n&$P-}zCX}0NY}+viWNgZ1 zRt=;519p%6`|TLL*XYwe6~xFTGU#_KJ%A|S(s1G57+K$&9`An9sc@sT{yMxG<-*Es z`sJ`R`jK93kv<|N(5_|v=&LG5TSf#Y?z)SBToVPcpzv$Gdu4?*u+uPLClbFjg+F=ChKqVlHY~2Fg*#&R%->pb_O2?dCWyAGyKja4g2tiEUm^~w=f7H|;j0T)( zt^*2MC$YO@v!Q%XLY_tp&e$&vzQFs z;9b*A?O~7MH=zS`K#zwR|0*f}-SJxd@^BI$a(?lwg;+&nC?EwSr(Dh7UbW@8n7u14 z_?96ar=$1f_vi@OkOYhzNU9h$>bfLT7SRL7-V}2c%7D@Dt~+x^$HqR`$H6uZM|-Iv zQeybNj=s@=jafEnkI({ifUxMlx1%OKzum;fZ=;=#U303JN;zv#;*JGck%csw85zg} z%w08C+oMy17O4HKVLKX%ld=Q(Dv!i2;*+)KU-no}2EBlpwqYXH*HzaaE)^5VmU=*a zmGInwcM3z2#|DI+FQ4nHt#)qql0ms5`l}87wimD@QS+xey*L2D7l_&1jeS8=Zexfi z;CEy?yVnNf#SY-w13V9N5=Kw5h>zReM4a8LC07%1WFcYn8 z7LQB$4VH|o3+l^QtQwNI=YgsUA}VKh0=3(8snwXHWRfG3{L^Cz?T2}>c-$5Ty?k6x zMrc2qEpVSo$WFQ6cz_JDW0Ix<-Tnz=s=VUdKVk+ob)>Db5W1r9*=)Qe^ zGpOBxMa0kja102Q(uJzzmU@#G8{ceEGGKBG%oT3kHo6dz(Z&nc1`;DpI`Ff_sj?+Q ze$MvY{o$!9#PYcRTk?Qmuv=<=^y9@U_jJFVoEbN41DH$Vix7M;EmUd!{d$~{Nc(w^ zVoyUXlq)r-?^zB(>kYc4TXiV&ne$QTX{U{K`iy1M;6(<(j#V4nE0{*TPC+q!mSkE5m_QOr+s;&y+;RaO`{1jPWgMYn9 z(It4MF4CTkzmO>+MN7&naiuw#>wL5SQ7cWwc>kFq9Rmh=;m8(L$sza;wi9yLoq60% z0237gaErOmkd&U*q~Y8-x-_L#-1mQ*fcySn z2OCW=EFKMLnb!uYLA+-GGDJg70`%0aGnZ5#f#G&;gI4{@mit9^T4%2BVW^FIW(8lE zX9bo4Jao6HD!^aVTwc{^m6#maSK`@_Rf{ip_m`T|Wwzb<#NwhzRzA_aC*G4#*$O~$ zkMjYZzNenNO+e~DTmnCyGAv$&k3$4i`I>6=@GwA}QAqN%Kss*Y=p%xEbuoL4PP#iL z!0X{3+4P}$FP~GK>G%wG5@lut4DES&?lxzK) z=((VIKD|GnuFvDhihD4wYcO;9GbeeTc!ng@FcSP&PMPE z-}bS)L0=ejQ{lW1weW)(Imusjp{(!JVFw7Nk2#k7_85v@eP0+Ve*L48X3bT4xG-=b zvHDd6{_i<)Z#71_QB#1QJzbsv1<$RAuA-CtR298EMeM3V&+(rWU!j=u znf?$5u2f#)s>iI;g1E<=6wg6Ea$Sqs>tJ#K=J7Kp3q|Jx$^v?heQFe$-fif|;uBJ^ zxc&e-4$PAmWc-Es#>Q70M-)C*7DLc(UMn;x-6Mi%UP{m5BP2Z0)C*TuKqPBr=ehL=-Cyvok3-U*2l1-ff02N@AZm7G z_0EE5%%0Z0wb~2SeGA9mqIa-(f^Rv#7JJ6~iBA?o_}ve%s6AH0+>2b4nJpXO(}qAt zdiWX4P4Cl1D+=V2vNOI~I3y2^g6_J_5&j5o6w$PSK`Vn!7Et#`sVrcd-A7|UMt@bj zLUn0fWV+i!Hg*P3>o3pLT2(rfA!Ab|vt>Bx@?=uU^HaO++i?c%-Y43eouor(pBzpLsXFfFa*|#=mp3diUilwdpL|ntqm?3#{;?+AegcazXspgjSoB zPHr5V;im=mQ<a)_tJk6@0fCWj5E;$gR}BuCw3bJT#5@xjX(O zuZnE!_ekUWd_?%4TKBsB3WK~MyyptWZ{~E`UkrHq&8nlP|CcjU>fGaY;HwSOzt6?L zJ=?qO#^-*dGr^>G_DSmJ;xoDI3f7z#(4t3_fUTgFWZ>3}W9^^ft($+571nrZYhpz@ zuXK-GBxQZYlfVFa8!Lb~gCKe6IcsE@aY?=%Cu#-!idY}t+U=iBGo+F!CwuQ%=CGqv z-fy98-@{qie&uPY#0}iF3K^y?eE2ennuEM3Bx!119@*m{$`|;Q=&w37P5oo=!@I}w zKa@U82tQFJAj#G{o3E1l|Ng0MTsB4yyw<%es@3BjruAo8lLj&p8}0OF_HK}MoqLQT zQ{lyPpu4f@ZVSm~O2*07jvH(xO6mSmQ_T^h$YDrE8BNAJBL@mm#3`n{ez6;40S3!g zq_G>?i=KW;%9=;_|HOLIfXj#Z^o0k(Pcp@~x{m2#{m9TUDGboWFr{Ao@2rsc0_#th z-UrN#r8@b*$&Ln^6Y!~?S@u=N%Wn=bHB`@4r7I$ICpp^Z1sEe1W52G&+@h$QW=|9o*L2PAZiAYU6Qt6xNlrIzM% zoGhp9@ii6Q0+j+yA{90j>FVL>2$=Pq{A=pbyxx=HO zLigXZ=lA~=*_bT8ba8OJLbM;pZVp?;KKxO`BLTz%YpD&Jhs{|%Y*X?Ss2uK#p5x)R zhCH-PO?y6QSR}yvy-jOVwXKWii?fr)SqbdgGD7W|xg@|6n)I;!ycNFWjJb|~PPYe( zDM-80*4$=gzf7Ia`siRceJ&dwOldqo-lg$V#sHDTzpLWFZ5F!!K)nSN<*LtJQOb%> zC`qqvI^Wgx02TQhJK9x@;6ulfmQnmv+ZvK~WyF7`BX4r)LlyBhzD=Vfe$oru;RXG4 zJFbRYUtmyke?tgZQ2Y|IDHV<%@bnP%AOm>cr|Q(6fGFrM<(`}0{}L6MoMF%Y03rdo z{#XBdI(w;a_3{@Nd_%3(Rh*+-xOCV`%yWtAdAv)(1-`eOXB6;rA5#Bt3sb(BZ3m%_ z1x(oXTF{2>TrXQBi?^w&I|1!-AjN-#%`YAS0b)0mDjryP!09=XrExQe2s0X=(tFH-eMTqZb zw~6~amtt_LDcD-e5eul?eO{3*%^@(WhNUXc`uDqM{_w7{4iz^~>E_eknI(F>*ELiP zd7mQOS^6_Rwu9@0;8wp9PhHLY_&-T(z$ zM!}>lTfr+TxG!i{+YSJBtYvG6(+CO3nIN)c&b7Ae{qHC&KB3agmG`&=$K9F^Z@OoS zANqy5yl{>$lEkGC9*^EcZT9t3hB9;2w#9#_GEBgK~4 z&$WPJWrMi7M%$wA7A;$}{d<)_`#KUpjd1#kiFBf(iQ{P*^8AbZtClL1H?kq}wDx!K zfUA!ywp2HUFKAx?>RZZ+r66&XmPNId=bSN8x~N0yzf9u!aR7%Y$DBC?Ob<4|q87+7 zmCrmFrq3EOVt}7Br2U3iK9Dcb;V(#CA#H!IeI6E-BLnuQ($JLV{vC}IKK~+KD*Pjt z$vgdxW-SHQ6Kp6{swHh2i9wZKoZ@RoZe#>nHQ{DOhSQ05=NX0mK54e?9u?Qi9 z)!2}#j>QB&h@b?D0=P@9qiYo;xHqU~7Abb#<&iJxp)Jv3@u#%N`2B4onK1$&^IE}V z18@;@w{!#AM%8%M%OiSb+;NbB9Sq-1>pv(ON+SQ&ET;2L%`9*A;xP`Qm?O z&8%dG$fn$A(6vT~){LMPQ@>X{-W*en7X3-0gMiGW@u>2ZD2`3dw?HCW9&reOwq!A+fJIdvpjc}eqkY>KZ1A^2N7HR*|5rcy&R#{~@oKnn zE8cmUMXxwRgbwvQ1qB>vj~gF7*M1b3fO9UbnQz7H#ach`b*^r2XMJv5`P2WCaOVc= z2n}FGZy;v(^!d!p&+`mlo~fFu!B#J^4-U6?4iNiAbMK9>zm`Y!1-aTZJPFnWaqBYQ zhfOT157jlHHLlo3iWV|ga-(pC5Rmg)?9Dc8$j`W-e_vgTH;PD6vdx659MZ-8z~IHV{Lu2nSqo@!k{BuSVeJk1Be+ladyIp`mVPtW9)=0VJV z5X?E0?+R|4XBPQ!)@6X6sJK@uY$RQx5qQ$*$=b z4vC-W`xHHYk7W=`M@nIQXF+W?{YJ=5bQ|Kt$ThndQa?Ram^kHkfaubDREYyX1`3b5 z7-Gvw+ihPp*>@sF%7RzND&64iYIf&c4PI&PeNnXEay|yj0)~H~5?hp4LJcT3Y)-1X z8le_eou9XumbzL#KYm*!(=|(0T3$iFWEFO8v7PihB*X> z4*mXyN_b@4usaSL<7B{XRIQ)ah-4{RNsg*>CUYxMug~o`mMsZ6S2fg1qb)o4v4lWn`Q{(aq2P%+)^e;plkYXezS&%%y)^^5u= zP%28P$am~d8k2*h4x@A0u(AE!By}7;b|joxt-f5^s8O{DeUgI7DWf_jj$A|N#?dA9 znQdE9#mAs@J}Zo}Et3K}wARdR@9pG=^#dK51(mrL z?_ayQOQnMFF*WqZN2c?U-Ka1*H^nq}bUs=74^D3XzT zF8OUbYDZsPl44C;lfTS6Vd6FC+~K3nog5>?>D!2k(U(me+Wp)Tf~{7SE6&HXgObVG z)`)nUc`@|V;q-=nRpu%!w%@~S0uy$e8S=Cr9x6%qk{gp7W*A{=H*2_ylBg+(aCSJn zBln5#O&i3I#yq*?ZF91|+3i>EqZlqRer)?yB=bK=S&8KWR`Z`*QVqj1ZNF?HLwHym z3|g{?#O}NiyS%sIL?U;=`*Ndq1(fh8hy;-Kl-_^}SyoP#Di;#Ep6}pladQancsAA!b%`2jw5$q9?hb}&@_2%)9n@OC22SR zV`ixV+L$9Ib+52&(ta`b-+(PZx%PwO_Gk7lu8F+x%pTYo8nm(P+x5`8tQUQP7KBh{ zqy(?e2RtvhSTN4ppb%S6Y3!o}{O1%-*M*?VJ!-e zuKTv@)+BvUpw0bs=N*=!RFRIF^wql43_1p+PUR1w1F{UiS9IH#wS5qRyJ~7TID}r? zzI@gmrL%(m4^LOY7FW|Og9lhF*a8a#2=2ak0>Rzg-C5jSf`{N1EVu;-?he5T?(Xi+ zUEc3`?mw6_GdYd8@cqK_B4iq8nqO>nx74 z31MDncIInL(=SXdoZNR*bLZZi5nf&V=QJF%F@#tyRDL)x>(@qMKjLoig-Ck~?R(^j zva3V1xTLR2&z)n*(44z9h{bR;LMb6>GGE}goxVYLKu279vmRDr#LdK=cQtnOAIl=H zUfqS6i%PCs{Hx_45|W2{)`qad^_*PsN+wEBRQ^zUtS@%h#wP}=!S*gZ2#c_J3NlR^PFqd|o;E2!qasyM&>;Lxt9?QSSCVOE+Rfo+|pmN|WWQpbhqbG2EAO5`%Z zH+M^Nm}c}L*`iTmHGbGV>v{!W z^ZtR;v%C2Y?NwNcHs;Js<1QXl<$H{z8&eRr+mD@hq5Xg&_qTc$EJ%YS{ii5%62+2QcX_eVdDy`zw^o83t7b>Rk3N)NzAG z!sH0&1Cc)izPJmcTmE;EN-QDg3u8t3?1^z@CmvZ{MJT_J)qR{cSK1E}5 zi^(|{yC3GY$E$Bkp3%RLxAOlptB<%WTZ`X60d4Eop_DUpH~6OKRhrrvD0*k0<+`q3 z8gMA7U~9aYQa%Gm>G^!B__y0Kg4g5mhd595Vq~cPJRJV_vg7@uMHo{wCG; ztL$V08}CbZTW&VM*{ShzDs$sv*=a7Ud84ZFzNIli%U`(ePDc1$d_N?#2D_Z+#~a^R zYeEUhg>lp#dOgQhE$bX-;xq#>F0#ti-hA=hMq-3? zyeaej8PeL|n2wliQ);V8%sk57$SB!e**uX}!AAT1UJz73(Fd*GRR|RY3OS+Byyxj| zvQay>Hgx##)%d_D2z4cP^RSU#PYMmlGzT}w76pnwurj$>Rv-OX{J(c2v~dkKrDoLX zA)sW;iUKUFOJbY}#N0vS#-+XqZ?3A4&GGd%*WKy0IcyUjQgnW@S~dI0wHx`|-1nSe zIH73+H4c&RVr7K0B$pOW*LXR0Sv!{(xuVBN2f*41E?sZai2rFZUl2B!)LBbqRukCZ zq_b&idXltxUp9*|m+j}wX7k6d@5gqd4)~I(v{JWc@D;P@oHurpYNCT*?S~{HEJLfMFTsM_#Q5(EXxK; zQ!5!J{OG7^V^~mh!shV%s9(g8^!Yr)t(3)TCn8xf-7B-5?>AJRgWwn|6eHurL;^+b z;dXMu3|tpzU-1j2e(|Wwh_rfCTam&-l?DqAo(@yy+&nG->n}WE4@L8o#L?Dk^KGfC zg}HqW$YtXAWTIMQ?7DEPSL9!$9X%r~+l*tMg!YE?;rzV0yk`%bZkRg8>fV)*JCfXCyhlNFiTPFbao;QO@VM}cdh89g8| zZ$qt**KHM>En(RaC6HZ=X$@6B5A$&?=&6;Pqzu8#6P@F_%TK5^7xjFqP&oSL%)ORt z=_q<9`K^-rHNSDG+D> zDn9Zuxk9bAQJ^Wsj>7RQ#UB3Rp&}t5x?_#^^Omq`tmg%aDPmN!ktIqNzXw5C2ePGS z+Ea$a){CHhR-H=-i2ZQ6n8ylrt-d+rw_@SH-^}d*ub2CdE*3Mo55k0$)NCA7R9lZa znX$4TC8ltxt2aU`TtweL`}N11I6bsx@;0ir7i7&=&U$1_#5xI!om@8VvPatpLjhmz z)pxN1h>uT~b{~{w*-;D9FO~M4+(!Q4Y-Dk+b1c{(dn&d@wj>`GyC>`(dKz*HVH)xh zB@-*++X^=gH_b;!htH+uu>Ea1uW!@j-ht2blu5+^xi=&Et$89o-M?#eCt2IQv;Fe# z{JQqhIX4{>I{Wdrs?pTWx%g&(PB*u5x~Y-vn{Zqb&`Peh`SP{X-L121L|F2CTKwR}hNYQ1ZCW7R%ofG{?6qx&IWTCsZZlOMu5awo_=JYu zZNegX4b9$GD39jOQ){{A#{R3fwZ30i@Q}+nHCx)kS!Jz#@Kbm}{N;&%^9w2R^ha*f zBDyu)EU6YOj6}zeKh}2lTf0n;&=Co#LAG($e9!g9p{5Q!qCd7O#W&;bkJ@Xkr#gPo z_U4VnG(F687T0^g_aI$e?1gB=2s8)Sm2T3;)z{)nT}vt8(RcV4UkmV9G-=O1;VLzW zHv+e!=^rTYR}4>1sz0alK(0B8iTZ`hf|$BhGeq8u(|g}lOFXdddxXW&w0if7vocs- zPNHf8_330ez;1bI*%z0L*Uhyn&{q!O+fqLrTanBDMm63Qq_bc1-|OM#^=4l+p2X&_ ze*;k1S6CL3M7%`66Ty&y9a>m^iK&{6g&Z$juk-7; zQlicM;!~Y|FBB~bY}_Ny7LN_ppGqRF91!E}t$CNNiPo=Lo&wIze69Uu;@; z_X3?Mor13ESHZG%kOE`s_^hOtjYaJHYBsI5+*T0!(wyuubi%F4w6U>`P+taut?(zI zus1N zE{?r}R{b#hgG12t>z=vbne4X;Otq`rR*=2Ud)N7S&ZU!?SVTh0yw@u?gqLHT0>8-d zkB@EtrE&BjbR3~OOj4X;%ozyBb&;kawARzqry<)OYw`G9n2Go}qixGvK>|h6g0T($ zkU6$}$k@Ml8k5S_JXYKqXM74S!PU)FX#mm-wkp~G%8t<&U^ppg zS#GA5At{t}>bK~GSsplKciuu=Y9^)OGf`9Vwb$+Ry{&Qe%e&cF0R|i zK3YaQ`lmwmQX7hCE<;mY0$RNsyeitZweK=!kWdo=@BnblZqF{@0Tw8yR$_wGFQ($6OV z&vo{}mbWR^W=Z7mX!qcsGy(TtwTYuJ>{2z8FWyZ0UU@O@RUUx3oRtAhMhL;?d4!z{ zWM{n3-fx8uw!x#}m5avZ{$8zPzWIU2)wSo4BY7g%hb;H12~#Qj$xeRF-_Z8(`Un1U zQ=xUX`ZNbcW}TC zI}*QJ&z73I$m{8yC3_lL)%^}j7%4h*stq~_V?udkUHPWGWNG>=FkUy0SoL`7 zCZKe|DhoS;MNrQ)^7>t2-0{cB7r2R+>l^3HA8cHhtSSseLUK1==PofRatS~}ZN;=) zi>0!SzO#jm7ygKt_UVFR?VbxR+>wD;4i7%a8!tNC&k#mEqV)Nz286ek=+Q=iM|Ndg49Y5o7!EKIz9ZWBak)?s$`RE zu|S{h+~gE1_o5=E5E`<$U+5uUrk%<%u1`r%o0Jeuy-8@W|8*FKDcUq-{c_vwAFYmh z3Z%1Xxq2()xPM+f@`Mn(NJ4!)wQNhzukM?}&bCT)<+V=yx^T~8=Ek6-dh<}-dIG&7 z4%nIDsJ++q;yChRpCQPrujgEjVxDM0^$D6b2UakT%Au}Mh1pymBG@fa9cN;eAxOV> zf8uaUa1ij#m`1g}`mXI#djI1CP3@*YyL*mB>Jb`hC8 z^UDL%j%)jOl`#-f@x&}Ul#Eu?@!}3{PM4lTC>krRKcvm3w_8;q*;k^ zA)LNUB5J;TKx6L!I(aZoROGFTvL{Lm7g44vb_hcZHB?ii{DwzXe0^gi{-~|Dk=HfZ z>nHg6;&$t<_7D-<7vxQzyoU6gMkl1yhf`;Lg>sj|?nX+^yl!GPTLVwV?8xCL?B-oR zozRaCEAd|%AFl$Sc8oyAMr*nt%N+P@r}b*mHcL~ru%GkJY~pN)04}-r1IYy0-4A-@ zF0Gu7Mc=ZYIdA?tJvzkkpM>y6OE}qJaezj|UCwIuZoShX{Wi38x7JzF(LIR3S&?7i zN@;_a{zC2W-hv1cj~F}YqOo!vs~8BOXg2q^R*m6FnR^zq`0a>F?8~2Ul5#MWJZ;*! z9PVH2l;pomxa)aAhKsM4RW(d!b)m+W(9u7rG(r&@Y!|nA~7AC#QXG$-ys@lntCE)30QV}e9CiRxV*FTT$PazeyA-zfa; zmerm$(D*rnj6KJvoJzd;hj;^Wsv-oIOH^%0fN&3cbXj|*I*8)Niw&VEl~YYW;z4v& zWz1t$UL>8uq_4uZ-IMX<#!QEKtX4s%nON_MYi$3GsMhKfXn!2Fu#X_AzOnsURNU`Y zv($rQo2zMM%v(M;@u@6oSt<71lzGN7IGCrX7 zTx`zkWN<~1GCy|SjxVL3^!6B$tki6t4T`ge`JDGU#MbPjv8d}I*WB-Xp5>T_8V_Fmu z?|Z@I*Aq>6-u#@5N5ZXqx;xmrkhV6-gvq9*P*g13cNNm{FAL^2hT(YdBQMoO%knm* ztw~^=nW`1?-};)Z@<&8>zn0VVs^IU!PUBP^%=Ku6{lq&6nxGe+@lV)eZ&<7&1s&8S zSn~vnZ?1-Z{^R*}IP2223t$tU>*bb*iQJ zG`Dz-frBLzsbOxbrcaBSIc--oQ5hb;hD~EGPsW> zx%AfJ2Dcfm!X-_eiixek!UUv0#u;63V|CNFLe#(~bBqc~hUx=`Q9yLg-!D3#19dX2 zOoYgm6gR`{hFR{pU?gQl5i8tzt#eU=jFRwrqV< zpxU9>`V+4a?Dw?o#q_SK!H;WEnf@b4%_QWTX0Wk78@_$#H39h{77J4E!#TXlF?0yT zXdBpSX0A8ShSVLfRmC2mJdR_Rm%Ewr{;T;Z|5_f+3irzqC1}zMiB>$B>0_%~zQ$*U zk7^|sA_!P2zuwc~;UK6S8GNzvzoN>%TKN^c!iSvmY3oMFp6XtCl4ae`bam8&) zh_|Ktl8(SPL!5Q^prD1Wn91^>IiHD+4nGkH2itwWfQD>cD4J^PfPgn}Wv1LXGKvl6 zYq638qke@pLiew4_Y2l6?JrtmTd8c}s%1!GrD~tMLs?MpqGU}cVDrtGZ(-Bzo115q zbgt+RAlw!G1A8G($pL6#`BjhvxMyG}vIEVJQf=FhJrUBU1Q0{CS(4r_Zk($e{3m|S zsD>$@zm#?whKee>9-}~}CW?bTDsSH7=71AqmZRONToA0ZexKa5oxvKYyMv8MLr}YG zUCZro0rBKJ->_JO9%U-O0SNqc8vTVs45si(7f^7w`f=vJ6om2N9&`Ta?uq7z?_qje z-_W4IYd0sewpD1>q!UadLo?2QHyxKvVLwCEt&5dKicffx1tX^idy@gnMVg3+qI^sXQBhGHOby42W$`x5ENp)9i79fBG{pm1JUYhZ!9F zOgLT-oDa}c<8aB{i{@SMA?ZW8*b1;7!ds>TQi%>%85_nB6>5w9{cZ_%D&XMv4jA_T zw~_T+U?bp#Ty2fwlWF5>)p@{b5}>G`^TXk=r5b42K--+#Q}3Ejo*$xHeJMrPR0`t9 zh{}Mo;oW!~gDw%CO~oR`3Iec3YEU`V+G~^Bt$_eBF@)4P2L#9$Tx=9=ypdLvlbxLu>SAmLtIS81W?*Zex_l(4IBRM~7C4u{H@ zb9qU~7Pfd!gIXOeiz^w6Q>liN6PawNArNN}Icoo%%GJa_Io#x)$8HbES8Hyb0nb+v zP>c4wt>}|GO8zBw55`rWh$Cqww{ECu+>gEK@$2*nlu2r!zMKeftvP z@Y+6G(v0YQMj*`t7lqiQ#G)E`7E&4qP3q7!|Jsk!D3JzD5tv>`8PJ*6u>!h##O(N0 z@%N@X0y}9VHd`8g>QCuKQji>kiDpt35|Z3SpjUbAi`dxzVy{@Ml?G%yFy`(srzU(h7 z54Z9!U$Cgo=tZr6oy+M!&XeW_!)9sGUM&eP>-KeT<<8~9zT-4kwOsL?-Tvel?qmUA z8*Kdr44=~TmD22{H58FfugYz6xkhecb#Ja@YrpTlBWePmJ5puMu-=a7G$8V~(ZI>w zjT^=0a`UPpJa8*kc6-12q>y@N`x^AIHq_Z8Iqpm`1{wb7eL5FUVQw$e?kjItW+Cwd zi8;nms>laf8J1m;;ZpJpEbd{aITFob8&z1cPVPz2i}ozq<3bWJ<0%!n6LV1>`xsK*|VaowAYWo@b8DpvU&(~RyZ~U)+7F2u-vac{2s56J%dNkHopPY;y|jJt3N5P zLFEBKf=Vidnc9+bNCH<_}? zRHr~o)FV^T^db)EyC3!Xj3M@Tvmu%wp@HBcHV;&{7r{LiF}aQZ^-9y|b{EhzJqgBw(DGEmsNa6&3CDUJKBO z4??!lsp)-GnlAa@Vxi6N(Tr?8K(a5&K{n(4AG^%E+t~2PTUcg?FAxcc2)+VEDhT%? z7r(!VQ3>qTOJqececQ=mE?Q8YxSM2cUf z>{N3g0r#Z3i$&5Py%TGeC&bGBPndl4MVavp89tSr)B2t1>dT{Ba0I=i{q+QqM&pN- z(M+C3-uEm8vO$XNZFU5Jc!nLJ*N&q}(fX)4I!7_6-=kR6J2rEV>Sn9&X{u`v|w9 zU$)*F_}=9_1Q7HaLQTe^c1B!=3{p&3XUhUSUZ0yx-}OdxiiRW~PD?*MzIvEmh2mO| zd+A(A959j-POhERirQE;h=9s_4_d!>_2b+Z7*|1hxB#ft{<)#LS#9DV5T5A7HD zQjZx5A$KJN!S(LzHt$GB+=uA`s~2uvGYNV`9ywr}rj;1B2C&?VIgbitp#?_9xAXl% z7GBM3yI~e9mU_F8`GROYERTy0_YGtqb8Etx*PorGPk34By;z15PcAy2ZPBnm=L*E= zR%F@?IsRv*9p895gdY7Z`YYR^TUE9hgyb)l&q(;B@TX;lFDrZlXay}RLemWGy6}R! z0g>e5hCji7R=Nb~@xn-Lk;7#b&vE~r6HTHNsp?dva9XeB8~IYpzeO-Pbn30seG6-u z4M^qj@`$0g94CfG)WvB7R3M8p5IRUC z{&lhcr3EIX8WBulw|oT+2D!pIjby*8?=4uxpoq2iZq8{^{C~oMpJE6?`+dID14AJw zt^}ayo_%}19jTZ>KHZeM*$}5$?^k=bVkUYn{J%L05?|qmv;0DNyuoYNk*GBBaz#o0 z=U4yUg5d%Lepk7p#o7l&d-j(3&h43SW`~IN=1ZpOw3_@Q27w^wLA{g2xKC%Y3H8bWIyH9%Fu>o0Z(d1O??!WcSpV}F|VqBu7l5H=vzE5VKou!+OK z4UN^Jz>TX^Ft3Jyx}%Md`P%d}85nQ>(T_*nOtbj@gbz1Tvz2g#dlkOUMvd*UE1`-|*pJ$jo}sad`6fX^H3(Uxj{U9H;rQ`2+f( zFp@R=jNr6PeVSaO`KeuwcUhQa&~`)UupO}gjRd2s zV^re-%>RyfG%KuLiPGnCwgh38)bG<1?>(Vkgdh6i?@4ba9EoCMyy?}uA;g+|mkmV_ z(pY%nkz0qU-G>1d8BApwJ{GOAfAsiu1`ZJ`;B9V2P>&Ww0SBq}`!=1djDm6TTj{us zOI-yJksg?|)H;i>R!+Xp0b|BBz1oRjoVkjB%rn|HtJGsLA64O_*|ac3Ra*GBCA zByaMm#dOFjeZ~Do6Nt0y%2(JU`jiYjjBJamI?uqse{vEYn*v%lH7o)0#RSvbam3s8 z{Z$_se`Xm5CbVb<*boF*&zgSG=lzFPO|SUkC?W)Q2seGkzJ2FIq{#w)&KZjodO>KH zq@-3hZU#3IkZ^~6WB(F84eh=MW`5t7XHZk_5W5;$(D4@bh~SUx64B#fV^f@d4|NJ@ zFzDKwL_n;EH_JfR%{o!ZjNhfmm+l6#BZ?W78@^(p4!dwMu$6wR?<-Y}U(;_id=hLYymfiV zATQtBwBGoE1yS{lMt>k+li90c^{okWB}w>MolT0l;r04%H5_mDuKQ&|ZkuX{ki<=U znH(ratwsd0|58wB5tPfdvJ4F5EL3{c&z!<7Dt0fohdMMt=N)T<*kbxE7V-MF2H&D6 zqK6Q~dLv#>7%PC6{x@W!zP9WG5!^7IC%UE$l>)k3Iwos->?tmNPmD<`TJw(sqv;iz z^}jw7Ol##H)h-O(Ec+BHEj@I&Njr!xV>RhC5jf{k-CJ6J?)U|*zGg7@`^d$8Qr{FT z=>OM$#eT&~K;r0>khd^M2uM=D+Q<&bgB^OS zj_HEH$JpYEr8^SHDAZ35((hk*_qy`;rk-ugxLHb*H@ZX%ke%jOQuKX`9TfVQEvDXA zkorxP2O28%Xct6j4C~hU=+Sja7wZi8XiwyA7I1*e7}CW~gF||SY(@MT1Yd?(bF5=} zo8GhOEKu4IG#`FC+h<$Tms2)Oao!w6EAMCcY;@QzwzaLMb7rP z3|?JgbS~#Iolq>zPAkOeRJ@p}w?>iAYD9B44{%Xtm-&6B;9}q)y-YWC@o1)57)H(; z7~ptsYWz+*7hDl;{UjN&aaO8=OLsFnbk@M*or26;#s=2IE38tBI;JsE0Q(lVTaJ^ zpaNN#Sy@FKtPQQkFQ9Y=KwS1d?;ZSAX)YptRi7am{hKX^Gr4sCVdeNK<~TmYaI5>y?p zeg>$9UpHI*mhL%ES%;9$zMa6=#{V8otU~)9NY+7>a(7t}Q%fV4ZB-~5|L3ewBZ1=T z+o8@NmR{JEr*))|AU0o-l@>efrS2Ej)__Ek!H<43hb>)t@e zu#f@=!E>%Vg)Tz;;6`iwVYy6bHd@naU^&kcqK>hgHT@f*%HafqtAm_YUenphV&z^P zBU&!T>G)YEedQ-c8Q(AMa32m@xGSrV&FeH_R*;OVW{$-75CAv-$?e;%=To$L>lFSx z<<5Lv4W}sl3tj!-PzT&4TTk7V6eHON_hcpVcvbZRY+iu|C>;(7wotB^DB+xx%{0*Z zRPTRHIMiZd!M0Bk1)l(vZ;zVJXZ&~X+wV^R^au-|u&cr)3Vvo7>Vs&6^Y zPJ|7*4VR^kdM7PFY+&+dridU5L-(Z&>JfOf5;XLKQ}KP;XO)XeXDZ`6tCSe1 zZYirTnLn+x(E-_HgSF}kX2wV9V(>P6a*KeQH}$BM_&W`ddc7JEVUwISLe38{qJ4dG zl~ZX8QjaRdB^8*Mc7Hu7u)_HZDjMum45439mg|R|t!?}E70=+a7bAlqMd(*hs++CV zoA>0an&S7k=gOp(e6ZEhzS9!W56`2nk2VcNfF?B`$b$l?7WzqqU3vb7)*`>3+!wfh z;G+B`7Zr8<^)xqSAb5>4>~T4T{`U^I!spRHir{}`GN~_cCj$3IP78{M#Z$p-+Z|50 zt1*zPN6NnbOb4LT!E0>Xc!BYu+!*JSvvWGZi?FHtVo?Qp8Q+fP~QPPggn?YU{o4 zRVo7pUZng?T&XW+B=7ByHODD76a5(GY2W+b(59yvlEeFOs6UiQ}i)=4Y8HKIt}VcJ=ybO=#Zd@ z6_8KC*1?-rPQDn#UOOyw)TMnWf|)87TWu?9;Lg{}y}+y+Snuhwc3}#QotIGSN7s*q zb;FFI=g2isLsV27yOB7GFa|cMYazR~JFy7kFb#e?Z1K8=Z)jt-aezO+Gd@tDA?kmO zUz1~%OaBfp%4eL@^yWB{A^7~-ZW2-dcbih%K&AP7Z<$H^=<(}LPgI=r#vP+vWTl{Z zEjVb?Y*gf>9ixXaH@)H4PCAKLZOQH{5+HMblONFiY4Fh2KSou_H$$k(O6Y8mP1A;B zu1FC8{0B1--&=r8TfIr1Py2%>Gvct_XT041CA9xJ99pFsBmI3#lo3f@Mo38X{K{Db zwSw?|NAk^l=Q0AC*%GwrT%BI0tr1A3PiV>RBpKWMHwKbyOZIJ$JR?$3vQQ%~HpujSU{Ni^t!5DhgsYB8- z^+E*ttBO(^0XH^{RDv&S6q5-lrlR%+7*cI+ynI!n+kcfq6#oU5qLz_g=c)qa#b6nQ z&zvy5-C%w|%5Tsdc5wdcz#PznRN+#t>Zbaaz6tsF5C_R8r&RP^9#Ei1X#cWU{rk`= zI4JCwXg;}s#ID3}$2&aeAE{_b`d`x%!I(DJ9AMv?@2b#u>QJtSh9hgG-orB5;r-%~ z(*A7r*dn&v#G>bgR;0%0A|z4X(cZv>pG-S3PN}T?<1Y@MH&E{&&2gjW)o_pHt#!b$wcP2(3(Lb!$%a^NT^!SQ09jAu|p*XUoNq!vqEJhS7?8$ zN)38w6(|w0-vQz&GVN{|K+a7QNFl)BEf_9pt<~s=UJ6p*LQh6i+?$p2>My}ck;sgAM%|` z6;M;NOagxp5Js6`CQs*-_vW|%pn^%LbEFgdnc3~wy@~!ibN~>KNr@OnfTt>)VIf*p z-mv%L0GGKNR$v_kg}(KFs(qqCcbXtiIcU29L6z)D{X`^fKMh;#*QA;(2jL}4hiOu)UxZe8uO^b7V0V3q5Q}uboALp4- zITzUvw5dI(%Z!5PE`vc_B4m*H@>%UAM+0BS-#_dyVLV3g zGFI~?$i^M-fKaz$9EQC`3>&=W;r#kzjzJC@$`uT{i28;nvA_r1dWW-*!)~h~l2~f^ zru2CK91PV$M&0Z8Ep$+2AHq;gkWQLW_m)`@qPJ2rV%T|$xcWyN_jMc6=X7_a?vGQr zX8mlDc241EKr-j+Uu19iej%$;W!b#yG2nmaM*XhH&&_83jUCLkWaiNGjQjxzS^)=9 z@Q$v1OG50wmpC_Xg$LpLt-_gaXI@CpZlwjEGq>TSua$^G@63?A5h|Z)dBaxlp;|6R zNjqzCsSN!TqKj5e6G$a^ilAWc5Q>yYz3tQwXCjc9h6)7YDG3@Y)R$WTyWs*`=x~Te zf1S2pUMl-jSCT9CY!9(`T}MQ37yykQuI6~*0GlQ>8^cd`#;xRk6N9Xu+&hxL6}}b; zSnI$ec`Qx$3YqqJeF;cd#TaJPHbNsV)A1OZt;cI3s;qki6jtM1NHXCT6zH4+XiX)4 z$k2hLh?Q=7j|2D3ix9HX)_Q7AyRsSbZ>9dCTRvjwv-h& z8eP$LEKvG#2xRc1Tjg_FK3~+Y{p{c$S&Ki z$SIOv=gX3*k^n<%YSI84z=kaE{iIjD!?~ToIO#qS#rQxP7ELm#g_2lkMto-+drh?M z$A5z%^xS|L+x`H*TbO3~!9Vz7%zpfZIg6pa-WS|wi)N6)<`V62y@9FhtGtQh?Gc&s zaqH(Pm$ir)ey(cu{iG8ND4Hc&h#6Xif{WNM29Yw=7jj_`Fk36X&H?)|`fPr$kw#Tc zOsO&NKKduFBXy^SMxkc|EgSac6#x6OOQ?@c+e?!jLHI_RAhCpe+H5L2i_oi%Yb^?D z^S1T`nVn4e1UB|z8!S6<{r$j_#Trc0N~ZAngh@8q>0_k-e)=Q>MfS&hP zpF8RHT~8aRP)xx$^8Doy>iatVKlpzgDFqJcASn)PI@AUAKNs8n`&o?CtzotTrrE7K zgBxRKYw+{euYqA1BN-U%C!X;7(cHu3X zq-6?m<9tpqw`#Cj3Gs1+z8{L>sEWX})5mk}@ETqU1vHdNeZpoK0$GlxNyLVG9;Rs* zRe$1I2^ml4{_U!i_HG3h3Rc}A+-IPE>KjISS)lrfty4ptq}6L$4=T5$gKwo8>-&aZ zq84Uc3Vt9Uxn;{F2lmlTS~~!+3hQw!B{|J|(%qBq2{3=kxutdKtezCL1=iJzAQ=d_ z85w=*OJQYca+`hocuFpx%@BO&AqkLHs#*zrp1C~mb~S6Gd&Jb}n9DqdpA`Ik3p!|c z#`{uqMZ;t|T%l05V693m0{u{iTz3#^h&M>-wB z$~~u(D@S^I#CL9c48oQ$-R6-)t8()@DRH7$09kUNzXtSGpceH@uw69d9yo8}6 zwZI7eUdr(x2u3Tc(WpPjU%=vFgNa3MOhg&th)h4l`-hmwm91X^Z)T6N!bFRcI-bv~ zP?_MZKQn>QORJDJ+3D2r^y)v2Wa5)-SL^@7!=xXmpM6u8T+x?GI%|^fKJ}xCcSg)N zc7Y8*EXA%VQVQ_+8sqx+j%f$3!B~9``}Sp6p1a+dj?Y@ECwlRslS~UsrPcE$fQ{b{ zNd?Coc>Sw?ZI4n;Wl~RCj3{IYO&2?&Jr?ToBWh@E;xG=YLcr(Ru~?no1w-K1599~- zT`n}yvPa3M;GB zBb2r?#R7byRL+>CH&!`7f!ZDdkyYYYRf3akn+DTH%r*VlGjvD~S2a|u@s|q;F z$miHjWZqZH!HXCtcq(L28(#9CT=`i>p)F}0mJ5#2o7IfmH16I!skC(st@_T<2xq2& zlPLG70*Z~Gs8JFXbo|)Wv8GBk=f&!Sh{VXDorDxxuY9<|_rgSD%-81+!=mS_`M<~d z#XH5MhZ6hRfEEFzqez#?Vj4xPH{zlTgbf$Z`%^)B_Hs9$i#IH6h>*y{Q>Pt0J_>{F z%bqni5xMh{nI~vO5!9{i^Q|1gx-X45>RBNIAwN`54Yf#p5~lH7CU@K{6%aG`Ew*7Z z!`*|p^<$gw^-L3~iT_uf%gRxf)ZPf!`Pa8X_W5OS?ar&xj&d|`!A=>!V0Q6c>ZAs| z7AnYr@?HX?w8T+kQ<`i3PV+jRJ{P*DLv;cSfVLTVK%|}pX3$tKd4tji#*i9@H z>gm|P19RDX=)95LQQY7G%y4e;3QL?F8@|VwBX*KPC|q+6xcMybo;H((KN*A9lPg2`YOtR0eMi+K|L5j3L(Nili2V0 zSCC*`^B!aE#mgBC(${Fw@YlOypvR2Jm+no$Vr$3Hs)#>?pv@jh?Uk9QZYJ*rO87$D zKjV34bIECNxTpnwW$weZ)`l@a7=;|Z(C}{eCkXJgCGWqw>V8a!1qlNY>J{&{!Runs z^1>TsMf-;k^8VcA{Xox0xM1PCv9GYyUE8yRd)b5tCeGgpgi+leQh4*nH**8WpRzYf zqNU&jPGB6(C(1)1un9ZWJ0H)wdDngy@N%dmuh+jTeU(+@fW8l>YTOBU!pq-J=-_b% z3GnZ+-HxL;12>0_A_fX%x`4`QJ?DM>*A12bQrli$_1R77&;8c)ua%2qsWnt8b>?iK&@y}ZwLHg%)c zoF*pYy4G6G;))!O!7awmBXkCpp$s2&71R^B7YFT3-5jT_N|jQ0XS?*&^Maysucwx!0Wi2ITKtMk}a*LYe~llW#HnHQVIHmm2BTo#qD0p zLIf>uawVN?!q?&o&HI{b|DR|HX!w!$H;YhTx}VxTZ1o09ZlqCTg8$ya(R`UVr8~59 z>Ca#r54&j~tL29%JjSDRWp|Sa-QWwK2;L;4@TkJ%`e=JTWe?fs!#<;u~!S;cq`ZU%*A##!QajpXBVe8 zqmsQ;LVi6fJmov?4IJP)(%I5HH<2~VUePBwbSMjVnrLf$yvy3X`SPW|2VhKD%=X4# z%4}TKf=TMcp|bkZERd6kjFgms>k>f?^{dzJXjfWhj|mLKDe(X)Lhvt#GhmU5W(ejq#S zxr`qfd*!7aB*FK&)!7U3W#^WgI6M_rd7Dt3H<3b^$5ph2o3lRK8jV@fb3FW6hxp|i z8$sK_DMBd4ZVBM14b%1d+tIHr^82!-NuxghIiQU0Hl5E+D#)0bo7CK{*J4G75R}Tc zp9EDXP*fQ1%vG`Mj?~!8#UfQm#`{O$9;g5NGA?Ga{su99+37p9H7SJUMk-@1{*NZ)~SbgiMFcWuwseJS2qW|&27 zqej^PDu0{n(SYzX#(h}~FO(+>e z;pO_@_T3x>+^>hRygvPF;~7d$#scL_FLDF&Pb0tH9^1GC)eiu)Uptvtot8Mh2{o9G+&tff{gHqX-U)IbqK~k zR@QC}Nh?51%La!mNVYsap2NIod%8ryhBuGNGcT!OPfjJn##k;3-KpbUMSJI`jv{@b zy3u!zyEBX-hCj?!1yi6uU%O1!KT5BG{@Ta^2V*qxY>BwTy2iOwLdc_L08RE;$;OVw zvTy?IrCfJX`ibue{sXi_VxkKh-T(JlG(_JLd8$P@3$fiF4Lwv*4!RVS?}*}*hg+Y} z^78?DxPf_b2Y{HJZGSgjIwB^#%YX8QP>R=8fr_O_Sv+)occ@Ssx%<;dj>rWsmuZBL zlUgMcS`4`;G#I`xJjvUMJrR`DOl*=sreK8UmkcH)?o_jT-3e-~@l`B-qKqPam~u_7 zpcUujx};Zs2b=>Hsi{EM@{taDjwQWoZz6~mOGDV0DIn;$*Rlj$tw~L7+4jN=pJk`q zY;~Z5;9ed*NoII+h;-?eF740$(z)z5B&(An+?2>7GYJwvxx5@Rl>2G3nql%XVNGGL z2fhE8?RKzz!?We&$0qM#e%b?^i#vA^;|0}7-#y6HxQ&Wi!!mQ#xXG2yyUGxPsJpH$ zijfm-(+pHKzdioVThhWt$2*!pLqe^{D9n=-!y(Nw&GqQ+1Vt}V|{ukZ+t z|Bt4xj;iwczNWi7E(j9R9hVeTy1To(rQ=FUmvlEMDcxPt-Q8UhQoo1K_kI6m-L;r` zX6DR3`|N$j%wr|{FeSz47d1CPvWnOkXC@YW4?|cQ9}AF~v>qUv`NT4;x>c@)3$AFo z1aHfhj6vCLNheN}2N}iFGNA!Q44|dhXgz(D`KT%M+DNKh`m~f!4cfi=4|{y8d?hPG zO&tnE-cVz_iUQQehbRqiv6p-P#S3z+uZJL^0Yp`;|DtL;31~cg zdY`~XDn6z@Xyb9!8rQp2jENY8ua@+Ap$>J;96D)ms3c#OI5khXBEy~rErELPVzZhX zeA0WcfZZR2Tt9Y5t&g4FEY<>?G%~!e@?v7?*SnEQaI+B;PHVMv10U15SukhVSFXDFsX9{`fL~`Q;!L z+nV?I9AD;_^K3{RqTA04vSbed@;fX?1t*H2h;WYZ1*cCWpaGU80aiD~TzLjaZ#dgS zORJY&c1W~Z0kSqzG7z6c8wndJqK^TQXg$F8Y5e!O=$jD$w$IFIN}b!qD+P7^^y_9y z_iYv2s_*;2d`WpXlQb%c?gBix5UEJxzJe~Lx-#I4xiD%}cI#F_?e%`|+#9V|Z~*!V zZ%6UkX3x{RKA0x0qdkiu76 z`ht1;zaonRJ@yc+Y^Dk`zv4nvu$$Ei(8TN&Z?<(#A@QKEV{|*#Tx>S!n#s&n{%7oD z0ZF^3+4ItLyHU!ju1P`Otx2-aN=;)J;Dd7)e(|ExgH3Oz8(^GK=)uI|LhVE#lfFYU z(pu_`8GP74jbOt+T6L48wUYU*n{jtT1ny-QJa!DcNN5Q}@FMcr2 z)!3w9XSls0xlTX@3A}JUmNiTJhln3fJX?5aZ(eTo4r2M}Td4Y(K2u>HURxvfJRdDT ze47w*J_plbTjj1VU7csSR%CXXId8pp0;NCi(-BpHkJlF`5TQ>N9WtdX|2^VW%}##C z%E^^*;8V}`T*^c*o3+%}@5aCdI^J%z)i_Vi*HCkaMhwaY=QpBoxhZ;9kxUflc8j*@ zdz3yGhqPozX2*U|uq{T)x7S;hCMtH(`^8zI2ix7V`*3Bp(G8zsDnBt07kA}z82|Lr z7jfnC5Bwg6*4Jk@{7E?T@X}RxR#W*SaU{Emq3d~tP6$E87g+RwCjLp) z0jC3ym}Hn^%w++G5G>-+WpHO&`sfM@T6Vt1=Sqam&$Ez>l~4Sh^8JJI4FB%+Ks{(N z%i~_2?6Ok@rjClft{mX91*(1ZJy86qQCoPF+)>Ua<}PyORC>n~qE;V^EUD-WJ|o#Q zxcppci>OOv4)Q~a?Tx+bQ-l1TW4R0NgmwZl-)-;9_2SAti7&BDBH0PKSg-89&wJ<7 zqS*t!wo|F??vEywi1Sf+CQlwjn1VULG)i20ef<=hzbmV>pVHa@l~L;wI$FEIaf5pI zHCbX{Cw3&ZU2BilqIb5S{T8L7s{cTK`g zZ46pOq%8viStN5ZRfD|DZrf53lHkR%Zru)YTDNzGDFuURY9ye<;obsNmj~yKn?uqN z(biItekd{4iTOdUu-E&qDuL}E#>|ZkVtCoNV*?o4mGE=O-qNum4>6dn>fhMMbW7 zq(7K^J|tC#v0pHL*+E3%UyiS?X02KJyC@a5z^dpn`v*?3xL=Oil>YfyPw-5eEX8`q2Sc!pen@tc3n=TyXw{kl!7scf;`?V?ek0O>w$c>vBxx=t2u!>+}sab!O|_j4I4ya)3!Z zZmgKP6yp6q`mPFFAF!FKIZGd^%R8epkb69udRwRzmA^(s-Nn)=CUt%U6hqsm+qxFtmS5(7aP8$fY* zhOr7X@5tR9pG_8P0ziUmJr{GTgncjUsi&~2uEw5KYlWhyF-P=K(oAs|3X5)BuoW}q zdzSFe9Rt{u(b_;=QwGpAOEC@+gKm1&7c6?+`yJdim-Xg|L9SiC<>-nO>3Pukd4|B7 zBQ9%#VgR}bXSXE+%qY+`rCwL?ANDrhTE^=GyC1&ExvS1D-5y`Nk!9|8;456x*<|^S zI!DMb*I3f-Zl)%X1jZ!6tXEEQ@)DDpI>t_7OSZnRV2t3hSNOdS?Aj$7;KP0MwJ=>J z_<_CgIMOrL&eNTAt2K(xCe%jL%hj!#^)H8DetV9O*dFRn*G;Tn1Ndo#= z*SbtQDnw(77_ApR*CvO*NR~`!2RPrz5M|5X{cI=;o}cD&7kyO6pJwD@(4yb-{T*?@_w_@Vv*eP$ zHU}yLZp>)ZFN=simz6$acq8^*aH@gZ8C3jby10n={NDW+ja04?MkzD#iCvc&U|;j! zNt%E_qm==!xKgvBmSkEqWQ}3_!pO?ecJ0@HU-I7#RoWCcx~(^KNqIO z4pTe4l&l4!q+Mo)$y@m`e?aKKT*d~T#nNd?}nS6cxfRmRe5YVe4<@@`0;u?-_+Vy8}!0$0u9wc=YB-p|K++Mk^%#(`w*IZ}`L79u| z5-mr}6!~4#*^=q{)9@SMMBKgh?=kWvDfxqF*IRllK{0vhQcc~KudJ}?T5as9~Bw)y!Ge&AG`-bpi&WKbZ zG|wJD_$AP?pmn1FA`{68n(7>0TWzAew+Yljk#AoQN;rLADrx()9v#jwC0=aFlH)|2 zDUym^yFQ5B5=*f<&z{~diq8Z=w#=;io+g>ov@2!r+|aK{9)N+cG4MvC zh$_Fd2v1-_#$s2Y-BjMktQ`CpE7tk$(&&M|;HpUgDLg}-x$H^qz-Rt-6^^f(`Xq9E zB{I}1LVx+#Min?4{u~h{QPAbYqnsTQLf(WuZuvYlt)Ax3l-v`NUK|ny`+wJ`c!ZZwb1-+#Ms1T!NRio&xZp&d9f=P`1v?3;(~6qp zi$1E>q_i4Uk2$QrUCX6s*n-&K5?I#?%KXN}&;C6)2U%jg)= zD+jC?%tNQJFUWmY+jDd46n_2$bV)L+9)OCHjsDyAxBegrV0K3{kjsC^B6Zepuww7& zfz7}JI&9ex&y#S^MbO?%2gD+Lk9G`Dz9cYO9A)p;*WW0{hA+T=zMTN3Rq{}=_AoS&ziuPE zY|>S}ySol4Z0KWC^qL0rhDp23X*aHsO#EfpCR>fu7mb-p{{Sw0^J`#WjYBDSSI<|} zCI)?$Q47;fZc=?7PVcgE90wL1TJ&hg(8O*FP=rz)L*!-;L|c?7p%TmG(l|x+&kmP6 z%k>Ap@)a`NBTVrx;-1h&=^Zy}HTJDS z2fL0nd!khzl_}vJEEAQeA~|aLD-2Y(ax1)yxrr)~miLvp_qP4x(u4-`X}C~%fXShE zXoCIq%#xb@69cfW)RAV*%;rhvFcxv~RUBh(V~yYqP=ySXvsXtNDerv^%;z)plD29I zK4~}Hm<+z{?H@Zs-x)8m7X&KNR*t$;nv>rzF^7{wFXwcs<3O@Y?5{7Rw6h$1&R-hv z`X!Q*ZNDoquk}2P@Ls{vI~z*c^x3sM!!cBW9}EoF14npGub@i7yZki_hQ#REGZsTj8()%N0eP|7{$(1ldeijZs3w}H z1^M6?yqGrUhe|Ros%!;*fh^oqDapA7&)&HKAZMp>b>o0R-XNmqR%U7#7a_~QD5i_O z<6W%4@8(P6bJ~P1r-zzm(bUcYReBaS2S7HyR!sGJ!^#Dt2n?I_44^B56f@cxU!ECb1EB219Eb`h*P`(X+f@2!A4Ga_SAN@rL zf|5U8lKQJ|KQSf zeCmv4#tCgfgsuBvGbbz89Dsu@VdY7kGP%Nzg2U#kB#U1w?7SQM8H-dV{bS)01c%L&*j#3 zQH-V^n~=PJXlD$4y9d*#)-qS;yZ>)Lp^?oVqY8HsXjob)9)i-q@#EGY^etNwk}P2Z_3qR~27vs_lSl;7r3 zfuHxwk@`Olrs4PBU$SY}O#y4OSc|+j7{{hjK2;th_3gMq5v~0W^%v6`+$81q^A&z| zXh3FQ;{->l`rZFPwpx95VK@$`6%yeE`7st2wEhKOc_a9|oANQXdXY3k~mqW`-B1L}3HCiDxG+RD3fBc%_ z&_Sm28sbDwcs>{$L(&e>!f&B|Y$P`}nN+i_x<=4s6g9ZF<7D&7(nn8Z6uuHhT_)wn z@=B+5v6^_dWwL5)Z~Sjs1ex*dAAcgfO@NckUu(KpbdOqs3r)E;e%0TWgcrlvUfhbq zvpT1Ltrer@QD^*EcL(rCFy8Jz)q5Q!=u8m1Mio zMhE~dif;12KowP@OrrlU9eXw3mrprwCw&_@LEPCdP5m-_VM{I$A;+%#ZM82cG^Q%7 zGn{6jufe^%?S-&@na1aek(wXdtLmyR?sSBeP*i(R{^zqmDz^9c2R3|HH(~_j-Q3}< zA{cR^tzK3IncCbAek~(k16GU;&8Ao*(7L2YIS|v=`{aI1WHp`uxd~4rq6eECQ6FpI z3&8n-A_mzSoPT#wCZ#TveC{1i5VsANE|9|?(LjSb8_WSB?U}BY=?#BJduk$ke^lUP z({Fp?m(gHUfZ}Dt1jSai9Hg_5^SX03e?1Mk^tj$WP)EauzxBfPiy>E7&(~>logli5 ziFnmtA=(=?nHacqnbhPme`ZzonZKkULMkY|LQxGcZP})^yw+|nT878RhcPSHttwa zs}f^Y7MJnC9q2D`VaV<2i%H#G2{S5uds?j;7^JLrz*hv)cFw?K&0~xX6U99@^#DLz z%rQYX8?VdnF43qO+YT}>u~u;lkfYHF?IQy=^C_mO+CuZNZj}fLXdV)V|R41^Ov0$H=H%&K!K*e=G^KVHy*E(+xjdveyncv?r}VPKl5JT*T(`4 z#6mbB9o7r}1)mlYqSpp}^q3X&)*0POgxQRCxZ2E(Ws_1vHe!n#fhuaA=NjaUd{UMr zISlQp8=Qfdkbz9u2epvued@J}!<);j1E2%xyG2 zjIsAnL)eYsB8%-D)O+DzT%)wApw=oo5O{1IWwk-XB#aTf{mFH&p?1M%w}%!hxg0`y zN>&kle2Iibf8+sriEEU{*yK)Jr;fWcV`PvIAW+SzLMTZ20jeUe*215WiWr=(r zkCM9XIRQ5>klm(yd_4(sO=b?ZHa7i(=?J!v^1!u1OeezQ7MX;_6T^yheZ*KOWhV+n zX_?ZS`*1c{+$CQ^b`G16`6CTS;cs3iVayS*E~P+l$56iXDmiWMxjD zd~vN`LX<0LXWZk{g_y~O7=%DByt@!FhDnpLcwn1JYJ-Ymu@E67U%MPq-Ur#$IkE=J&SMW3w4Aa~;SN<@NgLWs6Rr zgBRZ;KUx(8!%5VDGq-9AJq^^G*w}K26$`6?rSNn^sM`40Mslz;HQQh-)?sw5v~gHk`pU=Q zOI0iv)(tN~eA-J2Qa#H;i}w18dcPlp+A|-#kr5^TqM!g% zcnxHvvkV@kawk^IdS8h@1|B&8e*O?CUe$nForVQ2zLs1T`g0z#D7U_&ZEMRS)mK3= zEX8o?QbB?+7x2tzes0VuLV2Dj{bE0Ra<`YNyxB6l;RRY7Hs_G2d$ScT9@>P9|3$X( zX>s+7tY9JuS5?h}HGWnsd#H6obGsJ-pGRs8ij?G^%k zAQ>%fG9~~RfYlbj{w&^rMk~VhLGBx$bHVAgpAOL4s`M|+`&F&!o9o&Gs=$0p`M&O_ zfCTK$Hr&6~mRkD1KAodJ%@yF$RC(}T>=}q8zMzZb-B?2jnmnG;2xUYhnoo!HH2)Z! zNz~Am{>>eG@cdgdiG1d#k)}?LZ$RC8;Z~z6h`&xT4aMzRWql`JkXxs_Jx}T83=?>V%Dns6i z{RpB`je+~765kh>-590PG~U?r>GE#!5=AO8#~{8OHHS=`P8+)!0zl&#;|F^pT&U3_ zKn7dcKHjZ&)s#+9vrb91N1`*!ePXTKrDSnYz3Fr}&-@M9ehVD~g&Lr5Zq2n|H9`!B zl)TZS$KT;c*!&1S<*`qsXOLK^GF9#e9*yt+GTPvf+DXZQvsQDf+f0wYo38zQ;a2{_ zHod4*YcE9%_+iQ=Gu~C(vR-UC02t&htoZ0)))3M4gZGHfMr@#rX_Pgq&MgA$a?Bs3 z=xstb#yEOd)kkWDfYe`i{?MSgC!f~~(VScf;HHR``qyt@_3G(0PP(p@nFR-GVzF^P z2>nMj%PDx43BdX;nO5NP+Zx`4~GfD2+B*Q`__XTqB31GrCakWxR|yifF0juc4$Wewi5n+{1{Rl zF~uw1{;P9_T*RI+TtEdr5@{d>&*sHv5Li0X{IB(*xBHjHnn*CUJ6Z z7wCY?9^F-< z1lvE73J{|Z7&oV*nW36Z%zY-}^rV{hk|nyhHS~buzyg=vQ?Svr++PK4I89*Z9Q?(t zIs>{1}8u&}u z`}2~{+mgqCr^Ts{>M&c&{rZkyCg9YT4lO)Z(qybTyO4-Ms?&FBPJ(84JIC^*2Y(yz zg%aZT*IG<>ZOxk2lUaLCfHxAi`oaMPK^hAFhW0@$KKb@t3MxV8$z)G_b#k&Tv5u;% zO>WxoK#`4OF~e}Eo1m(<;l1rfwy&34_giHidj5bH{Nm9?q#DWw`UPlZaIALtZoq7CZbD&J?p5>qr&0!qd}zc1 z2y%&$6D}a61=>*I>c)17ik3NqZDY55F!(ys8S%R0;HXGD~lJ}%gFZ)LEN6ys)n^5neCT+93C6)f93Eix0hCj3zJu>Vk!h$;)nW}OLp|I95au@@+2S9OV+Gfr zP%lUDct};ZEetBC^v=f-5|9yfHDN(q*dT%T;ekXzaX9epprUes&v5oPw>&1!h6q7U zo3m=S8t!?<`aP=k2%K_nSjV0VXzvzI5F9a`N+gz|Qq<|Qi(hD>KNL%BFz-dlpb!~7g^K+)-j0zKexxJdeUgbDcpUY1&PJ%0L z7SF<=O{5sun_nluSDpt_>2I(c;F6K$&@Dx+uhSp?*97bl9i!O>-1tLNk*>8Q#blrx z#4EoVAiELIOsf0(T>Q2_N?LCf1s2ASS|hF0vU_R7P7M=`?F|;+b1vKi#pYF#=X87N z>DG=iee|yv3%s2bDgeYpcT@2w!vHkT_B6K_2fUs3p-)2pxN55~zR@1I%XFBm{_-M5 zaswO;N%Q94@0k1gas|TXUvmgL$<$LsRU@`<+%{^cKo~XLrE}YWQ?#fwZ|sz?x`-GDK+y*YD7dhFP>_EG zvsD$i3ZOY~v+#)ZNz)ygnL)7x_-_8jpW1136syGW!t_z~J=GkTJSVJPT&mO&n-bJS zKk&m;qz(mUFmdRzF$llPKn=MkS@1&hP<9y)$8yp$c&-7iq1^hGK#M^n!5=$+P$JKu zR2i+6I1bEM9pCqd$4z$|{MW=O9cZlp^A6bKxcle5Mvu>d?mQtI$!w1rjrb^GVhSc& zD>C?dz>BMhPUvL)yt)!GM@gN)-~l5>v7F2uxI^6&W_^8?bfAV%C5OS&*#dg+uFi)< zg2P;x&4$!b&L#M<_!y>*^awx$4r56j4#iVV{DA0KkxmjVIw>1!pY>Rs(zCGe=@}8{ z7!y{=f95AdS%prFCbsbm9^n<4X*J)^_UJ@1?n_cZ8CJ2eX>&MdimAuS(a${25j!BN zDAXeJ)B3xcAQf4$fdxIy*37P*kNUax$ATYjrr*R=B=c)+#`rl^NNBjVDZneRF1IJH zQPnfk2A~a2vpH!$Cpo^Eh^73i z<_sDX8cJ}Ri6KM9ClUIpSM=-PgDz+nJx0eR)sA+8GH{#Oi9FBb>c9Sx*nHlWEfEDz^r&>n75GlcS}jWyzKQoVjEd%>Px0UGt-X$1MSRa$N2RcCCX*htqDvk4D0Z`MNAE`SAx(i6p3H`tqr(akD0A{zFbVx8N=n`G@(mTz%?=^F( z*2E9D+dtt2*EB++)l4DuW*%)9>LpOR&2ObrMW=FI(+^AA@PNXf9 zt(s64s{Z9Wr;r}XoZ$lpQt}3PsgG!Y^3b5~0lMYG+B4Dj2)Kso$9>MWi4tA{q80t>tBA2k|mvuJOR^)B2! zBc#wSTU|gvVNmQ!X_VO*`Dp2~S7>XrBh8*>?$avfLB;Vz>OX;bYH>DVP$KQj4%I4y zJzFWGB4_6rr|df2*C^B*>zt^6?yGN^V5-d3Qrl4XzYEofdd*4#HWhL{{=bUT<(n=302+>e^{IQtGnmpQF)2P#I)S? zNs0bvbU}5Q(@}!r zPpc{X7_S)JcMWlz%#?IEB{MUWFMKU~H+y!e|SD`e#ODbnIF?_J4Pc^~e`1HxX zNx;iX`8i=g8p#}`I`zq(hRq8D%QdW9sjWm4vBfcwNs@=tgTTem`z9XOUl^@B)f^kk(L^v3D@NY^ z`4?TG%P^i#1g`RT?}AIIkL4AYO9SqhCjZq0tRp-~V$;WViXLfEwIJ$yE?|K%vJ-!M z=k4U}q%F^ld~xrhxUru7q@ac$9?`YtjTsp~#em0WbZ37snx=OnAuqPu&FkhcV3QnE zHBcm@Sd4dZd0JCG2wlrpAeTGF*^0rd%re!jafOEzVi4+Yza|`FLHH266@Z1Gr^>`u z>s%M%kv&l%pSuuI2K?Eb&Occ)&Z8=6DHrV#W>;$FTs8>&8aiEXbSPZYYR#sex zk>BHyzf**J?*^qv7h}sfvb($3nYin0cR<-bO(j_0Mmdg-10GE1!9WQ=-{t-r@Td)| zq#jh3LE!SDRLw@i_sNbrZd-}qqJwG~E7+X^5;*KP^V@r4H&nAk_GD7!Ym5&9qcr|e zqJ~j+pH>)NMJUkY^y`lSawobIi)EgJa@Gu*tJ?7x^uFWG;IT?$Z%T_q=G>Y{)vU?O z0#w377%_3|z@Ug)wD;m9sqH!wo zyXS8@P(8nGL|#y)CKN+Fbpn$8;BjcqggB-OX(uWRn>K6YiuOi<@X|*I`f6^Q-3!B0NU@-$m3@PMv)?Z-Hxgm zRcy@sdz|Zj=F{b$Mc9b*j>i`-J$xQ}-_1f${al}v$jW=QLME0)JtO4OlyMbx+rWsb z=NNRMl@Y?L4&Equ7m|f0tZ6WC3Z0O>m3cfFw>BjtA_?~$!@$l@Aig)t`=sPQT0{>uZ_#N=-vM5g13XKd}?yLK@3d_oGhy?mxd`kX+ zHa~-y#iM1m7l;_BkY#IMj56M(?}L3ZCSaDH>&WNx;CzuKH2&x?V1lx7)IumNN9QF> zx86y0N=+h?-mGd*t(_mWdN|@~kkUnZ>~n7$$Qe}RdPKN=pHtpj?BRFm*Bh3k(>hj2 ztWQDkISbU_3uF7y`dKsin_wD(E`};ADFdQoQ_fH9E)8ifwcP>geU1AxQ=MojGnHU!rEJ+(QaRS7Q5wN7fFRAo+@D7ru8~ zJ5J#yQeG7?2abaguNiD9HyQ8vs43w+ltG&6CmbMG)5#pB3rlY*UODQE8&|_Ughh2I z_N|t`^CmUrY=qKl(xDekIy+0Elp1$>VCShHOzUQ?4H7hPih}e>mo~={RukcTX|06x zvb%Wh-v&h&J=<|l^$9KnsAJ_e)g?D_%Esz7U|wxHC>;l7>&0oV)>`^jgdbu)e*1Wz zv!E!WYL6EkD)d0T(1ZCFYMt*-7B}`iVCYIVssFHyTnhzM)XAPD z&S78mg7NJsA+Q!-FhU*o9oiMYyB1%1k&>0mfZs2!_zw}C;RJJjl8{A!rcNh)J(UQk zcG;mND~$h}hP^h)2PjO9taI;&fOZvNXN^m1)j zGFIm&wWB|2&+;D}x=G7cdqJq!arQ}aW9U~4;50Up;(+PQqECZsUKWaj%SsBQS9}Qm zpguhX3}v!t!#%e@27C^cJ=5HzSulx#pp#Jlx0;;h0n>SfQ9Q>i=C8gzMw-CxRtJM>uXrjN@-GrHBA^b9e?l|pZ#asIZ7mepMYIfJ@ zatGW*idP{=$%p}Yo^mG_+5Tt{lWlQa0W}ZwxsHzuzsHCFTen2|4Oqkw{SP}Vkq*Zz z;SD+%akjI;^szd&Q(n+=9O-|*3v#;p)rX2(-`D-Y*uG(O%_I-@2vt2FMB3dXnh}vG zhhgznVTB8>4H6Vgq5OL9NZWMi*(%g{U-wbZ7SvwaEs+*A#^h-du-}|c@vkZ9!Kq*L zvcUAiJO@1FT!?@s&11^LfzU45Wwa#}cVP;p$*?RnB^CdQ^O?8EvLeJ&VrzG~2G&4y}zPAB^iC&v)ew<#PTe26KEzs*m6uGgOzajBR)%E;2E(r8goDH>_%RY*7ShehcNXjW*PmGe>v+j-Q(;F6B#a@4U9X76_S}#Rd zXpG$xm@nNQpHd5xDO+jzjZ;ldkiy#3be%pVYkIfRb3bsnub!!oy|p?TW|G`%dK^)# z@}9%P?MrBIG|~3oI8!R*y@sCwddOQgmYaUr-KM3oCx6k$n?4ARWWjCcG#Yj23Y1un zn#SOB2*jHrff&F3soM_D>inVZNg}z`Fxv0U$VplK5rt@VdMC7oIG6ig61-W$>0wYi zpxUh@_7y+_AoKF@qh<fJ?0`OSQvA#0K`F%B-=(Ni6c5WFJ8txwrQUlO z;Yd_$oR3(;ySWi}Ci_ky=e$tM+kIV6a6Mxti@1U%|Db8=3aXHXpG6dm(>2&I7L!!P z9obY5O?JjRD@k{S(JM6&?l06mw{^N8U$(K@%_{{-C7FDy2`6&X1W89iJ}sEraWOy* z(Q1z8I6mEymh*m)LZqJsHYk) zNPGCu-(S!biXIOYZ_K1tL@7s8xJM^n7McoE&VC=Cx8{d~25dGH+}96-zjLkN4Y#7M z`Ws+S9w{LvZC}obhVd_TF?%?N7-O;SuDve3pP80%0<+u#x-yy5BrL6>F-bNV#th3X z(KS0O324QVfQ5ZedA%%jac%_JuHEr2@gp`=l*mU*(7K?J>CE=V&}?661)$G2w^I}z zE1V6NGhDsk>r}7+{yP@;_I+{wd5y_S)6aTLf;G(5*$O;X9xhQX@ECc{(9tpchhb5@ zm~^;~qhWdB3TAoG#Xe{sxI?(sQ+En)IFi90JGZ2Wv7bBx1SjM6OP$#3K`{EM%=MBj zJBt%F9BeOMZ%bL>LG@%Y4eiN;H-a8{fKZDu_`A!N?%jA~-B`)X=R{`9otiSG?mTEZ zRkKQynkZ%M%NDK9*I?VvcxT_>Fw#&WfVmkw5{EEFehLSsXG`g{&jnk-@tZ=)nd3f3 zbmQZbArOmLyZGIC;G$EA7K`X~(d-&at^#!$8|tIIZTRRBK-e1ZzW4jy*Rl(=U!M^_Zw#7vyQXx1j(kE+X;-huPqmOVA)D8T z6odpOHB~cm`H=1VEatc}4HVq+OYb_5jm^1D3}omR(%_m|Q(>2Bx(pQGChd$O{a7wI z;@1Bb1mp&i2eI&;go=FAMXAj2XE4#(aP3;iviF~XJR?Uj2a$z$j5W!TA;a;ez+?^* z^Tj&_{^`Tpp+Tf}v?bJ7F3uA(f>0^dbc#9+sAk!j@>ZX-Ic-xijWWIT51ooEGrywjDA4(m-UH)jp;_R_1q>fYWBHnl zmCq6hG@uM1;toY|mv9S4GjrPbp3q!gYdi+hpNbi%Hj7hOU=+e6yYTM9i3EkEk+J3X z%kd-EaxoOc5;+!~dtX8{l58}c2FZ#SIXwBqx#yKyC}L)=8-ym(W!@vwpi(Qu98`r$ zxSD!+#Wr4!xK`^)*ZA9dT|IyK@HkKx@AeqUZM5#>$umQbE;zSVqP&fdK?`HR1TSm= z{GZpUV&MHx95XR~-gxZ62&zeTFBo5#9<9#vSl3FF51(NMNSw+|?mGVfMGA>0`WA^7 zR{UOK7g^SvveZ;baHwCZrYI&kSG7v8h2Tw3#-n zyL}KBU}yf?cUz;AZm&`j?nJ|!{rs6P;TipHGVab(Dln_JD75bB_!62bITR5dvrOcA zi^R3AJgWLv?>&hD<$CWZuN;KkJ9 z*RXo8$<;f5IR6XLjoKaAGL(I#hKkKD007v^;IRbbWXE+@`y|`Sa7g@)2RYP5xAt(@ zAscAtZVr&8-P9Z`8rj;A*~uK)3xD)}cXbnHWO3zLuzw+W_u>fc!b@rdWpf8DN=i=f zaf!Tf^j5-quEw#Ynl>Yr=%1X_BmIXjh3~Iy+0x5HAyWR(E<8i^JKbD#sI9*Fo0CS! ziJ^g40IMEnQfc{YNDXom$%75XrGx|?p}1FjZ$u%7bXGqGan{SZ$-ur_HFqv%R2S(u z7Y2H8CkUuFW~FhXL-M8Z7nA68QD}toi`{iz$-a+i+PXyay^cd@-J9JjDIJ*%?41Wt=2gnm zu=A8hZ(+nyVyezlC><5|e0bT4+M?yLHQ(QQqyax%JCCem`MHB@v;F+n&6PesU!>O7 z29GbK7Zb%{h|C^iWM6a_T+GzKGxJUNUuBt%-YZ8AEEpM7isQC(OakT31v{+y<3<^V z^cL3Pk}tdWb23I@Ce1ycuL2a3g%^;V3|cyfK)uj9IU5ykv%KT_mbT~ z26-n~l}}f_l(@__|564cG`1u~r^CkGJ$Po_C=%cZJt2Af2EDF*DE30#r7UU%0XOc) z3eGF^)Y1Pk*pYJ$TD-l_Hs8o7g$cKc#KjL3hMs^uM2hl%EL~MVRM8ftyE_Jul5T{d zC5P_r4yhp|q@`2B$gloEuYkuC{Ac-Qyd@BKRWoPBnzwe~)_?`&#P1{cN# zv4d*?J|4H4hyVR1kM`;)5T^idw{9pYHOVE!lZ# z*?ICLO(P58=_`7rY4w)Nc&4z7?%`6VYiz%oW;()0i~mscPUYSGkLyQTp_Cjn&Y!4VmmGB* zU%P^{??{KQkwo={Yc*KTj}+ja>2`iV9R%Qe3a@9oJWqJj^a4T+_7RS5Ct7yqNJeoJ zj)9EGP!J*3;Q^EZ^{#Z(XbcUB2a=yDpvr6={EgU&t}g0|0m8-uJ9w!prIfAPxD^CW zfzC6{kruxQwAw^=vgr&HfVr;>3F`W(9C=che8|$(A1! ze7Ov(U>za=brfN21@GL{mTY%~(!N`+jUkwuSoqWb9-_NEA1;@a1w|#to*haD9KGdS z;8kVyUF5ftcL*mEWb}X1mhJH#v}$mJOsLw`u=Z1&qq8*`&N3a4pFydvW2N*LH36=c za>Cf?BDCt@!-Uv?+(2%bG{}9XIZ#&BH-gbonmfXCIt%rwzp_RtMSQ_oDarjrb zPI=Dq=lT3VSp_!69AN z(XP{T@j_jz-;Hy{y6YqIOr<*ygtqiG|4y7MH4pQ2pLyt5mE&Qk%ATQiG@mV=nkRRXsQ=0te6Ge(~R ztRRy?zVD?PlJl%g3o}-EGHr0`XreQfa0Cn+dXP>Jtrp!03@fptgk5lfh+CaGq)z6} z*NC`unftLx2+=bijt<*8-a~@vBkv<*Orb4HkdE`sY4cw{e4qJqC`qQ##RHgUic>}Y zibOth3COV7Uiv`_4a~5;UMs@ST>Gx1;$JPvTSakkP+IFWZ;|0bwU0GmXo*z(2rqfrrT_>`N&5+4X&&xvcw@myZFxLtB|TqXb9~T@N^SdYcA;o{)q_;HStr))C;91 zeNMOqGXQ)G02L8QZESJ2krb0oSL0OltuWYmd#RxzC2-v3w^6w%HoB?y>1!(q|0^ma zNB}b!0&3z@B_SJQa(XOp~VO1lg^JO7FX z<;2G|Cp`sHHztW}HZvP&+532n$+sheroza}F4{FWule9(>Od+x`D>aj>(_3onSu{` zO|{c(kulL#3&OOjuuSP1%C0o!x?0{J4ihj2Gf2%NfqbN(SfZw~k`;ppY#e`dRQ#$T z;QW8(SfeNOJEs!{sBCYO;9Tqsr9fg80JBe-ecfc?2j7(d)9Nflr*YcMqZ-?vu1X=@ zBEk)`$qiR_wTLOt`PEjFXY=uTdAtx+^08;+zCcSqHWBT|hdl~IsTdhrPly2+Mf3%P znwB;m-^uSCo@|#=>^tXlUKuv3^jGCl<-8a8@PSBASBeGVw&JLi<2jDU7D|dg$flI` zes3k<*&JM;J|j#>Qk+7D^ykXB_(xX~{ntnk8ww=_*N}8^*QzMKiF{_Lru8R214K2B ztW}#IR8ikcBfR=i{f!;LOI|+O1y@liF z?eRJ8HJY40wcbDHN!W4~#{`TY&W?3 zBKGchcVew1_h~^)Au&GrKf|Yglv*7yIADTM7A|@|6KYNlimnGf1p-J{sO#$F5%3EX z0^;Cz*U3OAVRzh=DyCkiBFj55h?q#C_3*o&MdGubUloseIRt7%Y{)Q|dZ-f~%SA%h z68*3p7Y9Oqqfi!FTK6H`2`4CILA5)YDg&9;3H>{}opB}%UTP8cnPVqWLb> z`r;1|W9Ma-9$&Ke{rwVFX>U)rUXL8uk8;k_!wVD=xmNDosEbwPh5oFDuTquprIj%E z7e;LxW{zccAxd!Y=J{ob-4#%dBqG699+%N}EsyTLTrGkd=~K3cD?tVZ-wBWYS{PUn zvkX0o_=VwzoCz4bIM$*$n2#t4B+Sc4SlFGR&%@hp7^MM%*oVEqWQ14+CffFF$!}r7 zUqT(LdiG?L+sH6%rsVAhDe+UtxYMBqUWT8-Gnk<+i4wAKgDomW1xf=W8YOgcZ0 zwROJsKGO!yym)a?q4b&_^X9P1atU}9Mh~I0s%SUuqh)fPw6$!d1p58`15ViXr-Aac zAMzr@f=gQL20(g3t--r?6OXR0o?U2a4x_H}SkpB+VIke)4JwcF;1FcW;oOq+%!WC$ zd+z-9l3ja|65dj5997Bv%Ya-WghG@it66&e&fg{h9J z_rV)MeKlq;eXF62$3x^_-`(OdgY?=6#>-O*sg+uo;ACMXMos`)P%dJ>Z9^?}y2u+? z6tAu*5+N?~PJi#fz$_LNm*;Vio+9VJKz}vR`O&eu#K51XUBJs(A+oCDF8ksKUGPa? zMWgfvQm=c8aB$dKRNw624#=nB-jV)%kq%hDh;EPXzo-o+1EMx)FJ>_p^$`LvGIexB z;LF$}iQ;biTX^|U_sr9SBgAcnt@Gxg6^Ib^HZyd72wB3@J1|nW2wZA3pZL4f-3gXe z5_$W=E0Ns=Fskf#u#-AqPWDgx+a8vrSNU6$(f9ebM!t;1$xllnY56KuqX%ejF=%eHL*v;$ap)s%gICwd~_I$ z7m7iBq#F~GvjX_0nQM-o%fcOkmWS~T31KW}7+3=wL)qolb0V zP@U8*5|vKRE|Bw`Bs;#XFkkEO_7N<&(_AC{#n%bj z>Qm%l@=fPM=!xfM`{zQ~cXY`BykZAa(OY_Ec@5b~7(D zM1m%h{Ox!95i(dQpZ6m-KT(2t0$C~e&*GdGmlVY9w+}1#*AA2@$n!P*Q~FyQ@eNZ* zW$w4oxo*j)-$ae40(IhJ9z}W@-JLz^{n{M0bu4_4ynY9Wd!uq20sZ zrh(hh6Hign7URyL;NOA0t&hu&susP2v)>UDxY9>{m z1Xq}{z6im*`Jv;C;_xYsP`@I$Ljg#R0~E=@l>-Fe!xh)QXzHM>w#96dkK^)*V;v1+ z#0bxqD(4Obq*eY6?KB*wsGTQ5yvm{`Uv1b%exd=84@Q6s_$YQDq%d&O!hN1MxO8uH zXL5f|8N?7JJg)^t;;48GO5WU*O7D}_REGu5_WgbP)|<72QNNoj?y7E+V!%s9(4`(+ zeKbTS?d{Xq=Ru3cb-8(Kb{Wj^gCqK8vIV`EXa^8X%=`Djs_20&qp7 zuYmlstlkd$&bc-$?Swew#pDNDP4vB?p$jH>@pJB8{ zpL!Pudm+1s-X3ZZ+w!>YTQDY)cpqCJ1Fg@zBL?b90`biGFB2%=IHqo8gB1t(@m&)3a#$K=OD`c=?A0T8h^>^jI8fj zGCCD;%0@}MY9Y2c;3}64*U0;wM8uv$;iyrut_{@2oJ zUe=uIqMV{7_S^HwR$mu0t)dTXVxPIa76uD`MWe$%rN3@kvYDo)38cRp&!cMiEB z3k?d?6JCq7q$;<|M3F;?8%r?Ac%szr$?-tE?KY7nmIkJT?4j zS>K#|`wdffYI?SsX+$rhjh&f#Q!}ix$17;rav+?W2ODJYL=ic_b~${e`yqOLmEbv! z>aMmfMeFh0PBn*0{Zr0I%Y|qG5e25XTWrf+w{{^z)Lj+IpA-G2pr*$&4FQ?Z`~iXD zM1b%vNhgnNf9%XTZ~HznxU1KR8zVM#e)3pEC+UQ>v zgI)dBxvm_Tth@5@&#;Nh50q|MbT(PKQ3EQ*8j2)4I}=69jbExqlyXB95LXwM&1EM> z!sAqAWq2D&q;)7asYnfbZHaa;Ejv+B3`ji|dFJ_Pme;CTBH(rJG=OvP>~{Nq-fh)V z%jA9a#gKd)`q0h=dsJkPxT`ofrm6U2*5d{x9eRyMq!L%c^q+{ zTD@85z15S|I;PyeP`-Ai_cAU+hDg78i8u3J1glrbUO;p$ihLHJSR(ddm;tT72l z5PXkpw^qavoiO*7KfGVRjxASA{!9-Dy|iah|ItPHu3Y`~3_bSyetpA%C`(2Kghc8I zp`o835u~5?JXtpFSk^wx=6;>K1fV1~%cEttu#c0^o_Yk5KSji-Ityo1F5$~*Oshn< zua8_dO<#{Rh;|t4RD9eB(Bmd?NJ~fSYE~bZq@e=DRs#03KvF3!le46W1i=H>j#|ag zS-4qJboRG9kO;7HweVT2Mt%RiZWY?iR=g#buAeMSQTe@_SMg#jS2TS4 zCH{(N*=mJe`u4mJ_s`=!f^jx^=N}%qc&_H*%H!*rYLHrGke!!^y=*mac4Ra~6nEXN zBwIef4!uE2uo)G@_UwmCT&A`u>Y_Zrq!3 z4$G4tV=aSbGevyn!XJ^6S4-{+f?ajs$<0Q6>JN{GT4j_9zJZhhmzk~JoIK~6j4&|{ zkvEJs*h*UlPr;qQ3H`h12Cl(TLx5hf zUmJKAPu$P|GH-n<(o_o`KR>6ti4WDoo~1X1L>k_98DwuZwXqnph7Axg$ioZ0nE;+( z`j%HlpEJy@GYD2$nacV@aF!M@F^`pD*2(Q^SEM8rGdAwuCqfLHBzAd8{d2NYKbtl8 zPHLLN9BKu7-Jix-;Ih&GOl^gP4e92Gvn7q#c*`KoOi%2aN}YyEul|+mU~Bnx$1jjtZ{%+&|l$2UDD| z=(NbQ-Y}cs^(~JY^cq*lj6*edA(YF1KQ(QZN@42rAD7ikjNJRvm3jUolmC#>J{-RQm`aZX6LaG18dLG z0NDetS7u{@pC0Lc4uH9kV&$&#ShN#$g@jlxaeImBgL9V+>wq`4kF0%gt?{J;G}*Zn zuo*C^$qfm9L)?)*aMcZ?4A*@&q!ADNZOgRNQVGjxEu4AJ#})3l^HnZ5)7ngYfV%1x zr*EC*8R}uonf{eF2p7u>1q(sm8l2Z1K1$`iuSqnOh=~)BMj8BrzJ>)ltGN!mT<2VM z(55EXE4zUc$NA`&7UkqYg;%~T72Itd)u**)vs*$p*p> z^dQ$1Tr5UBXdg)_B&^xWb9fOi*D2%g;TH|~Yr-vl(LzaT7155TcJEB4^{ znUaaRI%zKABRaa@72-g?=$$y1}Li87BI?-J+s$)4V(xhc0m4R z&>5EB?Xm8{i0VOl*a!>Qo=s%vk6Z0)DYfBcZDwOQAt!d=toNF2r_vV=mE5T`goWC{ zFraFBhYlVWRo%Jstctqec=btHbV(3zS`Ae~Nccp<=HMb9*uob?sCbu#XRAa9hClnI z9Yb0-v^KI=iW4mnpDlodY9|jkRP`;HLp|nB+?BjO=!DC@1_P=W^2P@GV2&BbpHaet zOba|FrYyiGjn#Q<_qlaxwNOrl(we=DH#_tA>6edsM?D>?@i3|Db+gYAKk_c4F^kn8P6eC=ww32kILMOe%81eELAui@s7ZeA= z?LpRbrBi$rs7#3rX;?dus_+D9+@1j$3UNGkHe&~$W=H0E4 zJ%nGjS40vd!90yd6g=8F=}hY1l5g8)74`pre_vUdaEhwS4e+{RPi8N{wJ=PF!g34b|Op{T^_{oB+S& zyGFl?)1I6qO-s$vmg3GxeeP?4^6Zl8^zsYG3PHiE4!T}FtxZ{@NWA3ks>8POjuIgt z7Q1}LmvZ$DTYViNT|`YXrOhM^vGQ21<;45jF6hC9q02R@_4?7QI*Bo7SM{#2_K|H$ z25H!Xuhms_ypDrVkt_rzHYGDAjnBlFr#cWq6lshY3tGVTxc*WaO_V(TK6s&={LlHS zTl=j-)f}J~S+$|s`9n-~dMAI%Q;LG;4zOu8?ie_ywwo`S_DeK(m_`5%JBce2_FVy> z#ba>DtJy7a9~hW_?gBGl2zfFYZuB&K zOCI@rzNHXvig%;%SW*wC>v4)xf?Se}P-o%~JSyiy`wOPiB^hJmP!mB2W`r>xz3c3@=FQfusPB?TYm>zG+ zqxm5^te*x6_+;!%(+pWr!kzo~eM&#OX| zEOy%6XAB+=3xdeMco%7oe)3m>t?PLUa!;jEQ6x&hQ#E+@j42aYE%LgokvW zz}1`e{>t#W_ljiT7^3^oBsb(Zliyv@Em#-QCE6doz??tSip?tB_6TPyF0q2{+0eq6 zOfQi{q(i2Wo^&}9bxpILRWHQ3Gx%xM3BTRMeA~EvLvh`Dg$ff7mRhD`)z&vI{NeH*Y{_6<*dN6Nwor{69E`}a^Jn$M zIMul?L?iQR-h&5#41g*2@tymE<}ea(*Ep{qPU@F><_KnA2_pFd0jqNKkcXmDvFXQL2c`&kwuW63Z=rIU94@<@32!HeLQLNR}j8Wf30?`bH74#dh?>Fy7 zEev^!p7YR=on-2e-I$6-pyDN!!u!7GQ<)oAqu_C9BB5*cB&R@|-eCenErqx7%5t6z zAZ*Ozf`SMVE1O&hTv4&_(sr}=U8h)}>0$KHrlt*0;YJs0HB|bMe`r&3Wo2~4O0@S0 z>+f!6*zY5*1Jg~WWr^ERiQ7CEf$DH^A)@wrN29jeO{ri+$(I2#v$0u^6u^7PkJ{VD z^^~_iT9$$(Pby3FtILJhviDY9sT8<$Hs~We3*QI;(!us>7}k9#QTrqzAbFjQ2cLo%~pt&^TzbXKMJS)nS!c^&l$DSW@&e>{4YL8R4{=b@)y!&g+iyHjr( zD*i&?ij94@y_>T`1N;ZcdwQs0w(#xy1oP8@C6oL{3iE`iS2*(X-lOq0gBa(Zb^iAo zC#~U$l#Hu5S4Bgyrg%vnj2lry@ubgoSa%iGEZu;OeQkZ~Hbm_jMXwg$($_F*Gxb}V zBqsUiSbJBeIrgi_epk%{<@PR6qB7K*Z9+?kNArc~!`F9A9^8_H=Udv}v_zb`*r5q# zM+2GEyHEPc@N*$8NuXoVOb;8qC}(DZl3G* z$0$a@E8NHX>Ro=3ldJ72v|IQInMT}_@)~dW+?M8b(2`(X%#U0Ol%X^6;KZo0+>E{= zs4%llSS2c$igf=#OA8WKapgkBgA_}h3*$>LuC%i0YWx(zG2zuUBgndnGw*Mojo<5{ zgA`558n%rSIkd+6>t)IEd*+$-O^S@NI8`1^3VJRRxd)S`=h&7A)W`MQIv03XBYl!e zX<)+!1FxrLgxtRK3nZM-{L>=R<%@~2#EWLXx|jw3S~P^uTD`JO0F zE!4jsh%UL6qWTQyqL}exM(=-Trbtx8(HcXY{M+{}~oIx28GN z$}G&0qd;D#e>RQq?$eHoWAPvTCJ%sTez8i?>H(x!;cb)kD<23R(u^F_+HNYY$cC?^ z>m4nABaV{+6@{dgOoa>Q`*o89QnYG7Du*wX?y4~Lq9}h2i})>IRw#FQbbS!<^v?yQ z;?!rtG%;+rf8c)H*lueWk)~_tg5N!CB1Z!+8@N&yxp6&4*6MxONQ_FoIUzv8oft>C z$&p^m*jo_onVJvA1u0t@LX$Tq)N*V%F#Yx5ZGXt=d{oHN)B6Z10pTsq2~ty~iI;;%j6cLLW~bHTe39~EGO6y5xT%r?{h0;{@;zg`bUxEqTQ9^!m4F5 zlCZCtzZ~^B`3pKY#cX4))FqbLNbIGp?VHfjiEjggrJhK^c;yJ_srg!5yE3La+1tM+ zx>%r1_uN=)srcc_@EFjW3=~>7mi63AGCUc8Vn)l?7uA0Wkt;!_&%k+%M}3SjrT=a-X0`IU@5LQSQE^!{qAES(5Z z)~kv4Q=|=6xKyEk?@C+*_%+_WO;27z^HpW5ox}iHEOpcTL!W-4a{RE-MFIGk^T4Y@ z!V;cU3QVRvv@+L)^Qj!g#}iVl+jD3TjjyiWf3GIvrQi0uLc85+X8I|jrs=f;K%hIN zk?P13FC^RamJ@F9R6^O7)i*0^TDIyq(d(jN+ZJ5b`7(WvGz^oBEgPt%zCPSEmw82+ zzFCYA3TScR;76CpR>#1m#^cz0%NAziw4zV(WUB(E>MzKSo^XtpKIy*sLO0_q z=C*%a^}Sr;&*Gpyq^H@2L=PfzV(jJCG?;)qF$}o%^@kYQYA zH!v6#wIkzg0=GyNYXk|urJhw)isnZ=ZDF4KdWSC;oNo3bsVl&%G5+~{SBe759JH%b zYsnkxD8cc4+Urr;Z9WPGMm~#{nSzL4BBVogH)((|FD-8}TsWSvCEyHJ+YxRNWnfT` z&u3tLqUUEDC({OG_!bhBtJnr9j@hkjnfPtJH82k4fTLJPeadd_fu-j*m1F}}h7={5 za7W!ylju*n^%?FWYCX?+jM#V*qDE3vRHJ~G_btu8VIis>fx)6AMF z$s2Oi1Y-;%z+wXicY|_WMB;U}0s^KFR|(9jOc}1)gq`Uodp2cJg(=|YJ+8SYj?3gHaY0CpQ~Hf z4c;q3scuu>BNud_`lQ3flgdNY_N5G#TXitHUBmXnn{~sjl~RjcT)qg3 zXyJohtW4`pGnjOQL(`Mex#&L|5Dwt9du#?6ZB>*rjWwwF{_{n(E9-)!m8t#*eP9^s zqn91EwQwfT##bDN0z16P@)FY766|W|-%nRg&whXD{*R<>c$qxn9{e0QPI4ZG{O!z; z9@q8O&4WeR{I$}W;#}fot=q+q(b115fG+v_B3RY&6^X@CC$Uq`gUs3*g1=i6=eLX3 zG&H2*KANL=(Ha7#_uWKuBv``d|J3;Vmy2Db1;aDKKrT7vuE0y!+ z>zyI#P4@MU#4izYumGdJUZ8mP@nj1tA;1YMNt4*i`&Nvss>-dU=q-&GnxM;$Bu>9x zUq;G#ATC1r?)DUzh)U*NlN{FamV>}X4{rUku%R|~jR3rMc<*Z|tS%``U=weiRPg2} z(Eo&bSj@oWW~vodqJc|e=UMd=wf>xUAQUKoG+EdBuFa*>FAWucex1kp$rN7=d?etr zPC9>04PSgohL;McDJm-~Z6kT)0>V)r@oL^!a%Xlq`R>|q!i_B4`8KO z>3u10N7MeMt%g1GB+D-At%@M!Y)qUN8iqCag$!_!{I(zH--W0FwL>Q@t#k8IuGDUbUSmBY<$ z-?`et<=*f%5gPevMwVu&rGq;&r>AEd8Sr=4G|$c{VR`m`Ud>UItKZRpXG4&^YA}mK zr%jMX7Hy-}rT*2&z(OV5oN(6Wkyw*g`=u+!H>BU9jWc)@zjZ)-ez_85sAi7>>)2}e zfo3u_kscx#*kUx$V|%N{>nkj5+N^rJ1-JHCZzFv7{M0A)oVC{8@75HF9p<@vOqp== zHeK;L#?2OTjS=fKtYAAEeEW@9z3h!NEQ&eQnd4VQ>XO(7d1F?)$w0u-D;FtlK8$?{+`(a?X!OO*8VVwHen z?7P{-w)TF!3P-%F2v4*&#YdC90VHx-QzZE?J_AY-k?PRiYofcmi$(96I$YON(?a7{ zN9e^vaOiUBw9S#U_RMk&=kk7!fB)z2JYV@*nqe}uVmMi--5lFXj(2WVdmZ9hSn{7P zm)=Sb`njl^5Khmy!vLu1H701KTuyQuN4K>pGCW!Ma<=dSuAV}A6ZcQs69ODGJ3j{$ zhcr`*Y}HeGxaN9uB4mwxTlFLLZ7W)hD*{PVzuFO8nsM`By&JeJb=X2)FAeiW7ja)- zN%}h{w{2>bK74X>k=r}dqfR-N!IV=OM9LrkRbfQoXs$|*j<}*PlF3+}aJCjFd#I+( zy4Ek_L2zrF^>ZVdlJ-O1fgvU+?rOG%owtLWz;$;iZ4ehr5A&T@h%^0!Tqls(IRowo zHJcPXXh1SI?(^7T0*HrxW|w{>QLU_y5pHLG{KIh_aH`b~HpO{(3aA@S{PHWWONe!P z7}Jf(?vnMoYF1$&ObW^aG!AHjz>>GW zR{;$&FLE|UtUL(f+^d|*z6PTH6nbR2X2nzfH%?GQON*}1A~NO`bDeZku3;Y!7ipxP z?Vp-mkIz+ur5__(Op~`izmBFWw!?A7a;-wBH3xL~T^e-f*#cjvUt6)c{1B`o_l?&yc<4Svah24I}afiXS41Rwh@X;f%XwSfWJ#l#)X)wEkI zejbeQJ#PtW&X-J_SEzfr0IR#5m;wi-@k9~a1 zbY$dTvY{rttYPI8+6HJ#?@Ekxg4&=-R^(ouSSu4GDC^4IN=bQWx**w~JEOLY>5&wT zYb~bMtdL7Dk||V@1T;^7F3floS~K`SAaey9NzGQ}xF~IHEn{o++IMq{l1;C%(n_Gd zY@_(bfyCHJJqF=o)DrFj&wYP-8LvrJ?<*IOhx+)rA(1b>8UrLTWY^$$UoEiTAJ{Y- z-MZMJ1AkFdgTA&gT%0S$j2Z|O?PtB@!hXf5$gI^7-M6%ct!{hX3M@hF`EL_m`g?VF zBkyD>Qm1T)Hxbf%Y^Owtz9HXXrE|uhCJ%RLE5in1IV%#}>Q7T@DaIBCs1}_gMS3E( z>MzZH>4I(1#^@=LIX}(FUtVdjg~@vGSr`8sxY2!xfO=&ph~bKoHuOw`M^#H|6h@oT zJ&5$QZ%b`WW<&_iiJ2goWjD6}CZE--F;K_cquCZmU8KzZ*BH^@JJ2pQ6kn7?#jKxQ zN`_llhb82}u&ig0J-?Cy%^-0apJq88Jzfvk zgmf4{fgFm|m|l4pNe>IUrkh`)OA9mII^QL46^I?xDxS>;T6PbVQ4R7sDrY)cFIflX ztxv?VIG4x(tLB!s^T2!8Ml&O7x%pzJS+!_|c1py7{=rgLKbvQjKwvbkPm2Ul2d;~H zaZhfh3C_w{?ONTc3+lxo+Pp&~%-J;h~GG581`anymi3Q;a?JSm|{VeLt(5kpcew2IbALt}vn!it>{9#8_AMwjF6EODF z*JvEX0&$UR<4wHfgIP=8wUobjL|qol%mNAKn*&Q8ryDWewG}o{S(^HBuYr`nMm35xlv#B8H!xk1^0D<>~puk*561wY&HIbqj8rSB4M{5-VH@;Lasp3D}luMUV zPw;y)>PW%>WMSjwYF1uyY3GNL(sfSm={Ws)DlhIS_W(gv<8u1bwg8sg)cdfrKe4D2 zl>-x9{xfs|IACaBF(QfmKomZD=k5CO-;uGk7~$)^bp{nbdAMIkEFfmBK+tB9q1rNW z7F6A+5!w3uS*A!6j0o5Rzsh}oU%W&Z6*J)GA`)$+2NK7XS(&1G{xsRm57hoTTjQ!e zCO>f^I}c%9P0mZ!3uA0;3w-h8+hUfNhkD%EAqPj$-TH3LS7dHJsN@g=_wMDHHX zqN#3<>n#x?SL#>_Py+5YtN%g-!?b&FD?b!Z_0tbmpIhZjpu*Oy)$iI3s4hDK;;uNW zwkf0HaL=BYfSxWAtVpkad&Tjc1NYh;S@6q?RU&Lfd}+*8ZWUB^u_XalEahK z3+Zje84MMVr3fdsZGGoE>&DKEupjbZNhi8=*QhO|z(&~tXAVI<3Hc&TR{la1JbC>> z-zEbCzb8C!okl@WEKr$!O(ur<#7D>xHLKKbY+;AVBy0cj;XEfZ&WX< zeOmiGLV{>+b@d9q5M#zITz&+EcmabGoqUlpMWdoXifKza{`F$`?Q3G5cOH4zu~g*) z*66$6ydYCuuZn*Hj_7v-?ik)m$G<$&&4bORfh&9Ov|Y<^`%=yR>snt{Rp{{U7wT|) zI=6^QRuJydJo@qdKF{B9CO6RcL?!JDx_=@m`|JQzSVKKXmim{Yv1w3`y6vzzF5{EW z*{uq&C)NG&Na&c+P>xH9K26?CfYd#;cXtA`W!fs}o)aKTFL;KsH$+ zZ!=V{i5~rvt>AR0YY98I$kVT|i{&x#e4o(PBi+6}4>?h^_WwLe1n%0u0)|8+42yfl z=-{Rd!1EA(3Z<8Cw=KizypCUn&|}}3j0!RQTU7-Mw~kzuTa8He_Mj8I)0N8H;v_NF zK_R3huv+h&O{WhXugMKT9%d==E>tbY`zhp;Qc`89pZl4ms0xgCO=>CMOD3`3)P?A6 z`nM8E$MSmNNTGJB#ZGZf?SJP6WQB2gM8v3%e!YY}boF@z0e{0`YWy2OBlv_ZbqZ+% z2Xu!tGfA#fS&)rVYH5U;c-kDl4aZ^w!LN-5DE4L%I`IB}FbVB_(k^1hi}*KEz=Huk zeU4g~6g!yc$LwbN;q3_JY2NPCw^6PIZ*g6N)-Kylh9lLVHCB(xdLsWW)KoOE8TK<( z+`#;pLPPVF4~jq$R|3ONFI)) zGufTMbc|U*S0pl91J=LrPa zt&xisLz|xEPhG;mTDD|Z!(^?{ykBscA@~)sXtn|jVJdqLW!rchSo+?1f#e&+MNZo zS<(EeYzP)~J+&b99I;kAM7SLI93HZO#c%`MIf?!J|EdL=U+z+|pb-#pL$GmEIWQ?A z&PA4ARvi^@ZpkM`|2xR5VJh?4(eG{PH2s$T)GuvfzXpNijd7%|Dg)Wx-}>+aEo^ix zTe8JN&pL+@g!}P|x^z%7f$hQdmrnhG%X@H77XwDGC`2sk(~I+wVBGBw&M{KVt5ksV zg$n)K3tWkFnuU1+Q3l%ap?aQgZ2RFxobWS-)tJfTZP!--rSa#(ZH}c%%YSu}qPSi+ zegpC0htg8tf!VhcifuNekC_&IKw^PM)8IxRUuKzUfhc4xPCzsEyWUfs-=Xc%`PN0D zcrbmTwFcIsmtX&e4M*}BGEh$a3M9eyFR>_sN&@RJl)jhg^O6V^Ewwe-LP+&yWQi6i z&OWLO4as+5~OsIa#LSa62*ON(iw2B3RL_(DpF+Z!AK6N8Yjc^Jf^ri z--PR!r4cMO)(Pc*jfyp4HDFpA9v5NDD@u$NA>0nkn(uPB&e0o~8WTl}g`)W9lk zmf!YgMF=tb?UOlbjVU(Ruyy*%_Wt#;yt0o+*EYpjtYlnntVODiw9Vamqdnxk=$R8B z#HfemSCxz50lvqs^5-MRK&T&S~6{E;-B z6yD#x+^`IwolVYz&oJJ{Qq}?yi!9SMt_m}5rF!K%>x#W?j+pa-Um*;yIVe>lko|mH z9XD-}{l6yK@E$NxeVLD-S_}N93f7y6b$?jC1go6gto`*V!Y@GoQZdW-+4H@zj<0PfHOj(`t?a}26D{sZoX3#DpCR(aw@Ym zMl~}gsQ6F0kyoUjI4-a#tYByEGj#%5K=&VzY;prWtd2@q=+9)vYw3*2snH?&rTFa# z$AY*UUhKR46w)2>ET9Qbbkd{vr@KPP%4XAD^$1Y(AAMQB*m|+h0HJ}`>?t(ROeiWN z(TR}-{t(nNl1Hjm;;8$|gT%}+E;RJpJC8QfhPxv0U+-GDx@eqyfV<9tP)mWOpHF=h zE`;&#Y%&n)Y-xbBviha3Aa5bpO!J2!y_Y}#X+Ehw4SMl^$>RB^SvyN}8XV$vg^X-# z2gFQxHv*fXJ6C(<&9N;4JlHG*^w7qmE2*J|Jo%sFOEvku7;3MJQu~SqLt$S?b&Wgv z^`t|B$28HhH3xK`@D(-NM4rXabH{ErCQ{kEc){AVSd+4_;#dt#fo;J%{X!&H(!Wjw zR*$!E*tEheaKD<8)oCT(Ms{;UX{u{L2se4*u^sw-u6`95rl(SB1jU>r0au_5;>}{^ zE;#u39lO&#EgyAb{wl3Z~>EjxAp=G4BA*f ztSB*;B>l*cmOqBGt^dZ*Cs}EtbCY}{a$G*to-~YoDAbraaYpXnHO57+=m-x-#S{HC zQy;2p!}a&=-61j0s8YCbx*tu(<{1fAxyMWMA1AM&6LC(=d6v`)=D~4SKYjQz7dQ}Cr+`vZ&=HCrd36|94`Cyuo>u4 zJn{(dBV_I4Yxg2J9V6S47#;9BkfD28ACcKiX77RaGa0OrRc z{bh}6HZPyxExm18Bnx~cDNB9vTbgng?x$0RNIu{KK(1|FjK+PF!HxYzX?*~ZYxs6K z;HkS=1rZ1V%Db&TAW0{pyMQyclmYI_MU>7n%sP7-JCm(=wU=>gI^CR&uQhoFr+B@uP0ahu@ADA;A>yQRPb@c5R{oD+a8S zEE&k1rIO$1fSzV)e%v++du$Nt;I79kTGooY{=K1#DNFUm#}K2YyNo~0NPh%d{R*1| z&)s`bLoSuz>Dc^29r+!zJ)t#O@2M?qO~{R zAGlkfB0F(V{7}c^s7dBmjjZ{lZ8@!~JQ_$LRE}|xSa|AN_-xP1z$1-YYtR&Ld>m#= zR+4=O_aCKaB1c)kjKfu1++emzGVth82S>ssAL_SbF^=T;Kg|Lq9^>KoO4xgd>s_=J zMsr(XiaQh{M33=2gWtU7w!Sjt&F%KNo!~D{)L%t7``I<)Te$k1s#m^w7-4{Bd$Tl;Srs=K5d zf2KGg8hmS)&LnsNGxvQgQ|^fq)TK7@gT~^9fnA$pd~Yjy{lC6j!w|B}`ge8) zVgN-k*p37_0`yiXD*2wSVILkhZunK#tF4+pb*TG}zg zvI$io<3!ZQXNNP%zE7-klA+H5_LQCq1X`~ZD`H<~N5*%Rr`3~X|5~S0|9v^=_ zPK91*@^RS9=}uB0N++{gjfxC#AJ27){axs6W}Fy~HTx;>ym6J@UEMj~EFd9a=5Ek^ zW~w7rt5${ta$T_h_(o0O;_l*|n?Azy6b4mJdz#Vjt5EC*1OuAQ{;3Z+=~ciJojIcUxTKAgd~%X4FLr#+2EIMUHXbZedC`fHk~ ziw;R#kSE^Oq)oPB2CmFnsVGb2W&J#HFg+pQip5$E@p_4wFs&BnG z$$T4mK=5OM-B`MZhcjigiVJk=4$p#@3h8^OCJQeQmwzW<7cF1#pB(gj^#kr{a`G;4 zl5h;NYNitBjWha#Nzy)ak{9G;elbJ^XYA0Q977Qg2x_) zyaUC8zjj46_34~Cea4OmGzqa$6ACP!b;am5p@I*^MMvk2{aLI`!pvb6oqX+<|6%8F zgarBRt@txyeq6Vqm2*GUX}PqBVWKC93(TL18^Vd$>JD zJmG^v5n1dmYvGBb$89Y|;tzY{8)ga$ZyANaE!k8bOn#-XyHCNMAXDHmsNVwOCjJ9K zc&$l9Hdhqd-GpzaDo8!%;;)&Q)>(#*ru95x2<=mt6%BhxeIl5xT|Vg<&OdGT*bN3+ z%}`hoLVy?)?ND1t}6Xp8vXi27@iWv|< zjkfAr7iFuJwomp;N3kHQh4`Hkia20DcZ~e*`oh%TvF0Qs`GF^t$eXzPR!mVY95Cvj za^JLMN6VmR4tjLHmNjRXS)g)Q(@4l0v7`;Umr~-_u1e~3WB0>x1B>7H#Ky%r!9mL8 zbw0!K^P81pd2YMaY=YkFR%!3-DF?rsNK?qvi+cQ$Yfr49$q&CL9slxr8!u@TVow*C zPw0X2Ty^7Wf$EvA`QJW4R=Q*j5MKLY@*U7C>M4o&eHm*wbm;Hv@aDC`tdeM*L`#x_ zQgO8JQ3(@=?pO4O^Z(p7x4OlaJjMDF<~;Mj9<@r7H0ggq${O{TePA1|M%KVhMyVn^{3I+`mcER>hLie=?*0v zt++T2EPQ~(VHhF5??0+v`+?@B8Zk;hQJ72L-)2x4!aF(8c&fNoJLu@Py-z6O!ckrG zJCUBirRG;A!OXDu@27Og%0aP?Dh}O>ugoFT$hd{t_$YeU$f!lAt2fRbwb&c7(XP{` zD^`l~D!Ikcj}+2Yc8rY(nTZHY5j$*!&Dru`$Q5XCcnnUHcItxDkx3?mf>W9dt~@^R<13Tcqmk|=E9l|p*iQ)Fe|7-+E9=_J#MZe`I?lkZxPs7WN? z_)Mj`oZuAam+#$a4WTGlbPa#=Y;I-2grb3Ps~UrrU@Cw{5Jz#wbCq2koi;@3!Tm)j^a zfh5ZVy8ff>K|eD=Gre=pj>2#^fG3a94{G^XS)m`KY~Y3so>pck)Z4S0NnBmKbWsIq zYY<)9@cKiyH@7-iH_!dv#_r603%q>YWp2M`n&r10J}Vcy15zd>;@>_;qdI1g;3CSC z)E9iA*WT*o@wZO4K9ewaL+nijHihPNOG#`6-^fmrVklw#s?%!&Mu8r9-`8dew%TLE zoH>1a=Lq@o!8!eVAxpsyIkjlD!>&(r;n?(dYz~{z7Tr6aJOye?)KM6!UWscIVi6a|EOz-VhfaQ{X^K^f+;TI{i7j1 z3VWN!oy)N*WSK&Y!mv`Vfk)INmxyP<=5^{~9q-pK&6pgBzS4;WYGlg$BKHiVKOAbW z7gA8rr$MtxMXHUMwZ`e>^rxZRVn#2QH51f7fPCTKLQcGt55s2&FYFK z46bt4j4aRy)$&>1(n-@Bf33T|$VC|ipGy|sjPGB3Y|P>zPv3|niE}`=W_TU(`Ee@- ztzgN%+f1FecaOn!$>LXE3O5>t{^8of5R1wE#JdO?C|AdFszguT2^T@ zS$E*!N;z@(&&JxVx=YJCEHIZdy)Gpi0o~VElpK51YTjQyP`^5NsqUG3K_vb+_0QkB z&-8kR+%+o>uQf}e7qsDNLLZ}y>PFHf!V9orHCi&+@Cf1(`sqc~~1R;Pl&f8<7THB#97^ zvaf;bj0e@7Bn7bT!(~L@aK0EyBJ|@tQ@rcgy(<5nFHR0;-|663qAv zEy<75vVc^Jwx9xgzC8F>XhE*nn>oxk)abcYvmDx19HQgi!+hzi?#VSkV=C^|d^+A9 z$spHKCa4$e}t?MYaTi+NkGai?!g^2&At4?a^SsLxYP-g zL*Gl}b}J-&RNwK{kZ!_5NL88`n^S9MyO)+|tF+Y)(jkKRC!|k5gEn}|2sCk5>9|%3 zwXNQZ`cZ!7yFpF$MGd5YT{g)=%s!HB8#O;1=-H`|wC?_An~L}NterQAJA7^?G?ISr zYy$Dj0$m<*q395e2vly_ZI6dK8Nc>rvh6VoV{r7V6#T!j*uTBITIh+_3=Ga1h#AL&NPj%#V6mRC2?tBf$xh& zh^l74_uq53A7njS%`1{{l*DyWVQO){Zf~1%T|!D;KFbGsMnza*aPcMI1w9*h} z)yz{JY60LRt3a`K@7iRWV-19YdTdoAqq7`v4uAE@YO&CM6f3JJq~Bd!Vaov!1+Gv2 zYXllv*7s(0e}CxUfX{c|wzGG)m=vtxgk~FMc0z{IT^@N16*Ny)x@|ox5+<1jH%Rd{AjUGxN*uZO{s}usgUEI+mg$HVi4S zC*~Bwexc^F-#asC%eTBToQN7=<)t4`;gI~N%053?N7i#7JQ=3+ccuhi(s=sL z>gB{!AaOcVXZT5W`=3TOw_=U#!uz_mvDZmdAV_i?4&St#S2FyQtZ{?Z>IETazb+#GohR_VDSl_UY3Sr@kYpuaV`pojQTT zbN>Pxy9nk|kyhEPkHSuQ;(YYkv^?B&dFrN>A0Q({bN*K5>tJ{ymH5$Lq8T#(34_2u zakwW7-V%GY3Y}^Al{it;?A%?GLQX$u&`-Ilck(;V&e`mdp}%pX+*)l$$Fl04w>EO0 zloSta{!qsko`-*hpRvKIAe<_%Tn>&JckIE5xih|dNV_OPEu`2D zkvyw)W6K=X3f$hM8_}Ecvl^WpiCAW-M=ac6WYPWE4{IX)cP@Ltxv+jJ)-nN4+sX8M zo&ECOy9hK+k3!&1=ukQu(Uu&%ggLA$fwd*k9H%)9jX51xJcw$9_=Hyotbjm(zJQ`1 z=fwNqPJK0P2aV9=gl`6qh5uV14%@|32*OJ6nqQmAHEopZso*1UrEQ3R2`giThP!qQ zrmLL*2jwk-7U8>8?Bad$`mWVR^FlB0gl4DeL1JN2~sGpT6(14{TA_#Y!nIC~p=b@|>d zsA$t=DrnYWi_O+W3)x`iKinHx5N!OoF3vrbB8&`|acrmTKD}xL(dSXahiR+g6jn2L z(30fEcZRVcDtxOZG#Lo5M<(S=2+A}@1j)p*Re1NzM)&jUad*M?RvgQ24%nKooCA@* zWel^JXactd&B%m);oDavwc>YHVOrRA(d1siLzACAZf_=xTue3BYTf7VsRwK`*=xB| z<#cz7^grm6W1rCuNKy;xyYsLiUkRC%<{M4=7qo}7`2RUq%t zLJ4es`YW}bij8@IS3(C$E#{ulAW<9PVQ!^cRSdUeyD^coj`&PHAhDs~l9Lu*@1(lA zsanrS8fA~G?EP`sB2{=qly>hp>BEO#l4oT3j)gi!5qTD?utCgsk0dW5+o^%)L$yO} ztHq0YPR&3JRTnk-TAX7g3cV-`7!b3e)zr9Wp?%lV)D4&9Tnd^PMBLRaKj}ULhiZLmiSQS8ZXlUR@Wck#a-#-a~2Z+bem>H-#ne2=8ud_nc2_||oJ z_QdfdsUOe?sq&Y7b+4=WX&>y4RS!^2yXWGQ)LrL7heLA+sqU)Cj+(Trv*R#E^~_S! zm+e`Sm#I9lkhzI!T&3@;5*YR7`Q3vZ4wwScy_284jI55GJXGU@HAc_V;AtoU8^R>A zf58E}IB;q($pA&+)=eUP#;F&vWB9-Cu+Ur%<(MjcoL7w3Qny=ieoT*TaM=u#DY$n& z$rdHzK>QLfvN%^jE=ojT-1OD9zH#|2|Mk`xfyHg5zJ)U8=5;n)m^+qE4IiPCg3)k1 zWJ*Ltm$e&fX{27aUS}K+Hoh{wpZ~s*Y;u01)5V))%E9&XA+x_HdV8n#yLeB`^4Wr& zSx;7YMrZ>WPoM!G7P$1?!0V2kZA?J}$rZbt;T_UceJXz#2Jh8#r`+q#*VS4M;+zPH zZ9xgt8hb~$|GYaIA>#oK0zpC~)}#JzfvAF2x=G3MgI#(1$nNKYyupViU3w#Q^LR+k zbM4TI>T{LARfVCoStEgtcv z{fEXBZvC^DmbZ|tu(*%gnvyPt#LRy-R&~i4Cr~00SBKa2WL?J$}LFA%OvetW%TyC=MEX0wt5&m%n(2dv+*{&{=kdKSj-dv`Hj zq&G3h)=PlMVmxTD%L@OlVwq$~=ObLcLQtS*?;+!@NP^Znz=^pmNGyT2bsYF zqw+b?+Sd#*NGaGJsMS-9Gf9RaFV)Zh2NVDWjPN}PefD=oLvqunVDVPHYig4^rs+ou zfoBHb)>S9#&HJwnueq0f`(7pPOt1ME#d#G1seV>BGzxQzEs&iL{K^VGVtsSRxl~bN z<&i>vJ2LrRUlr*y#2& z8QjZXoob+iD(>sgi&AaR0f5^1hOLlDp1z}9x5_u((ZtBY?hE&3fgL(ebLyu2>WCY^ zF9hxGrUzYqu)Sj~0MQVzP;zbTnb&E=s=^alWA~-sFZW-#R{`ke?w7@oGE?QL&?ny< zun+s(d$|_Adq*z|gG(e1v%ZLuiz2Q-) zUC#GaK(Sgr_Q;oz48Rp0h1=JsE)95kp8R zS+EQ-s{7b243~6Nu*aEF%Frh3GXB|k@kcX$4niOfXH*}Cc=m{|CJ7N|BgFrK2{0g+ zxnXcR(Vlr(e$!Yg%T8O+_0wBB%^CGgpLH!&_$3@(0QSUjH+@tFR0nM!Js^kKXA1F6NwK=<6YT_ z@xcVMXb>$e|t#VMYAZ78SLAwwMqBdur znFRHt6xmC5qb)dfSsal0NX~Jx+oINH3{y%<&O?(lN_vim zF2fA5?oNkp`#(L`pZv)af%B3ZMKYQzDVH(4G<9NTG|l8f;I`Fh!ZG%9=XUN#*w+cY0oJu93MgO|#~!15jBQO?G*-e?~bv~5ow z*|Y|^u|QCI$p^;pQ-OhZgGyxswd_Rt{>7>{TZj47w=ySPuse1&>`#z1MX#p*Z$!X( zOCQ(EdH>z;3 ze}zzf`gbE4Q#~RE6pKPjk;`AEy6vfl5P1zD&p%1SO>TCqRk^w^2?9>@P}fZUu(b{i zBdp6!cEskR6qg5*FRAwE<=OZiLu(As(+1`V*Qx8J2VM*fD)e*5A=He9-o;4i&!PRUt0ET@~d?2){ZW6j3f(*G&g?TVq#Thozr3(G?l#d zkfg7#8#|Er-gSo%Gj@Q4N;1RNWP-Gj3CyR?N^k#jrn~7LOy^@xCn;ND{UJ?~5u1)! z0?tv6MS?~{iXWCaFQ-bj-wnwA{@Hk4*5tr=jY``dy>1wvN|jGIrv4R0zxgYkc|U^g zMPYeN9Q4-(C^mnyURrnaN2Qgvw*38VD{9sRk4jc`;Rv|KV^)D=u?tN2RE+m%Eko|Y zdvm83#0faO#5aM8*%gN+X{4I)2{-F*N zm(?g?6sh76|FRw7$-YT#(sI&zzZIPP=lIB&lym*|_&sXaf8BV?gMuimsg=lig#&PM6_Yx6|-}&ti5-zl){{dc5Emv@l#zv~5mj;uhGH6!5q5 zM!<9KoAoADXFJ;`(<>=BBkpbK1I}*w18*cjj_{$=V^#xuD6vbKVhGl(#naeL2Dl1dF0id-F-rj36%|5N61r^FS;6nzs~C8K-afY+esU=4f8e)Mm&;Gmt;`VtIv8- z0sh_Wzj+5#2V702c=U2U=15Tf?2U>&-!Uvm zp7T95hRnXoOz)BTohv;BD)_p}`(urKW?F__qZZqVpDB;U<9}3#L|RFn7?l4Rv@czA@CSlS1H3*eV2_-^=t$`sY6SntAG1Qd;9pK z+jE1M=cEJt=yg=7%Ll{WDSy4P6;zQMS*+Ju4fy9>DqMZbDZi?91B=RZsmesQo)TB+ zmZ&=DFNaPh1qRwJWUzn=TO{v<#91Pt`Ti_2%AcMhf1m<7kM5@FnNz)frh`th0$ge~Jwr8L2Z`}E%Y;cip}35auMi9+oZm$qphQ=s zmsQciQr8~3=J#%#J8cLb9k;Pd0xRHd^Rx?li%m#>pEdEGTA|Z=)o*IfjK5gP%0~B{3=L8O5})S`%843SlC7=R zKP3APsW$*{R-+Jqb6g@ahn^5N_wpUh#08ALp1}jF<^81bL1LUBAFwGe`3_mzo|&Ac zvF`jn5aYaBqx8zUK)Cxhe$zO4`F7sNq#Ap)H5r|a^{5jO;d2?ibwE0!^^W%2AeyZM ztD$&{2nh-dt?qu#dNmCJP?p{}tIUBS4_$P55{~r#a$i>AjCFkne3i3upYFjjJ5Pq@ z2j`nDbNQo`V&p)rIKLVbTGASZ+uKJ8=~&B70eh^*Q}sLNr9XiEa7N{)sx{MI@ubpf zSS(Q&ZWYvak|K#oPyus~WBEiEaMx%ZwECqrTvv5#qqe&Hhju?s$<)UBDwYTT<_(|M zR(|XM)~${CU)67u4ooCdvwF!lZm*SUYZ9;pcRrea-I~4s_z5dx9Wvt#B8y zZu>9c=V%C-0cRPEm@xH>~}Gpi@{ zx`BNJW@kcaAJ>Xb`+U&!*N?1nzx@ipZN}{D?UBqlwe5PX{i|?MWyQ`!|69_^B6@3K zvCMSh_#Y2i>)#3h>4ea68EoyG>lSV|G!RpJs5qB)G+GkX0E=Wl-X_s`*9f>ByZCQ3 zq-hL3w;KWlsxlb~v^lhw-&IJIs6f=U6q#%!kJ|5;k<|-;cU)$r4~&+uw|OfeHgIqc zzM333O1UT=tx*vfViY2yH~f~JItAsUN;4s8{JN7ZwO;gGXJvh)5iq1oXzf0{>N~}J z4f`^zSE$bm8V*i1<2d&R+(2GZC6zf!((W7~J#MEo@@+%jGUq$DckgfNz$bE;`{ zZT_DzP;AiLZ_61alV3kxw)*eqfnq(FT7Zqv8PlVpl(8g(GwVgor}Vnag|P^Mul;A7 zc|R&6PELM>Tr%df7lJ4{1S-T-ALb6ME`92!8mi|=^V#RjPEY=(>9`k z*Fez@cv*%umDS^Hg#htz|Az_LP(O2J9kbtoItMpbgoRf)Z@uUF8Q|Zjbq)h-!cf}t zG;KHiqgm4Dd0X5*-_sA6tHx5&64W20_LKWUJf#QZsKZN%YPOr?|7-eIRt&`(M0`D0H zH0}JSx{NN&Su~j3qgNgA5ax;+|D;dwTpaJYChBQQ8K>?OCY|GpHBWSpUCXTlN(1dx zGr_0%9g7A1nOl{8j0cLGjO{J5?CI&9#DXX61^yfMf%&Mr6u5&pbR>U{rtO|%oVHS> z&=&W7l3B`P$)b@it+{Ui$gy6^A$+ZS(o0r%Iekil9rigu)2e`YX2LFl2}4^IUqULr&xRxpVlPe-Hyk6I zJg4f*lv8$iJh7#!WkMMk%3426pzIiCwZ$?qwzM4>^k5{F$DD)8^17!Ai}bHXhiL*= z^(465Ym4iFM>R~DB+*p@IxQDvN!z5@;+n1_LwSFWg{*!OLSM;yUQ|!4Q>Eh&GgGaxO zepjPXRC)knNV8zBvT<@Ua*(YJ*QPxbsT)Ta@ycFsXc7Fzn zy1pcWRY^L7U6~G#68ponu_v_iGhoBaVJw@E1L-%+Qy&Mq*_#Z@(VfA7Ysif{tCA1w zREV|I@YIVx`zvFcR;RQL`S#ifA;S$8
)Y44X$4Ru(DSRK_cCUzj*whG&Y!}>qeFZ(RVtG!oI};lULy> zal|z?RE*x(;A}dn$B3r|LHlag&o7yMEyC}WM z>~i&SudJb3Y}*46GR6^=`;f|>6FQY2S`HpVJazn={W3cnvcj+EgEB2CQg>KDM`M!0 z#n+6@-q9%$0{|N@FKMMPCE%|ya;8X6osc`3TiwXcWc_=n?vCwjYFo_cs&PTDm(A8$ zj8igT-7You`zsh$a!DIzws4t|B|Y+5Eet%5kQ-uXJiLb*t&+;K$Cf_C#BS>!S-I{1 zj7Og!qddy1s!{eMr##8{3nWenMnP8bIx7?3bJ;Y0X{Kv>@xmGDAC%TqHBXrg`ApaD zy={8*3Uau86oJ}Q9XK6t3rBkfxA9J;JX7YVHca}lu1W}BWWnc)=|MF682o`O7tGZg z1B)RowYwubiWtn#rDWAS3lNDvGhGb!ulNx%4^kW*=tg?O-rgGZ>+@NdmtN+ZL1$zD zWl3Gxvas0EwbZmg>DWGh8Kdpenr=A-RmvRpQ{8e>5t1qGA9hX8Urko^e1!U}l=8-g zZ4iCNO>d8vn|63;k=iO5UJ%FYxmjee^v%bL%hU|>*4=^lV9as$Q|{3Bsn-?}LKo`1 zAHo)Sfk?b*0@-;dBH-&Z;CC`Zk8!jXc46(C#*)xSnp&}86Jh(?2D$$e4!yk$SLSr_G5JMqQuKd+Rb5SNF``N5z*sWG zp}|@?dP+s?EX7AVg4TaqgRqsMsE|NahOR`G%qfL;V~~09Ob9*sh37t65Q28bIxWak zG=1BidsS4uIZyv);BhQO#lTX+3J0uk#gnvTumeQw?QV9O=;W5$w?)f<)sH zPB)=}qh*Bbe)z8kSMlQ#)0v?gUChksG}xmF<{4u*tWC_RW7r+OB?8*^5Xx0r6)hK;P#n9S!lLehUd z(V}P+Oh(nwlA;*2LX@J+Wz`<+k?D`96z z;1Wc3+^}@HOdjmtKfMH?3kR&Bo#XI|ooAHxNvMx)yUo+s;m4+QlO{MDb_1H4c2x=Z zP9J6hkq62fS~J8sT}*P?xYr0-OZ4i-PR6XP@?}va{&+P?Tx|814kk{<7F8CwC{>tk z4liUFB6g(#=~U?mMheZG*hLRJoH?u+L#1yu{n^|Ys?OJZ42_RMBjmwP%=U$(po@y- zH@c~q!v82=9|O@w9qlv35#Q@s*dS-Ah1@vJ)@b>!zabOp%nU=zc*Nem>t0=Fxotg= zX@FK${Ps{tcL02$`F%{`Phxj0f~#c+W=K>r#v;peV(OkpZF}GL{Ixcc3eD5c=M~pB zMo!^@x#Ch*=Rr1dbO?*X*3Uj(o!fkqcS@Ofz-?n;e7{-!Pwe45tSkJ7FUy)}4~LB4 z>_TaHUPHTcGIWnm*O;}XKYClsC=XJ?p#zq5BIViU*{3re4$@q@S9`Zw@AFVRcFT$w9x6EO^9(OuH+hOwc zEcfGHc^^(A^fFWWX;3M1L6O46Yi({OzUw`D<( zY;`(jrYRQxXiPA8tVkolAC`n!j2aigpmFGY;66Nf?)#A6DSbhRetq?|k0>w-GbASD zQ`#~GJNPhT!Og!Yd-3_@WI(7}fWZ$DkeYirZq7Bjc|qv9eGyW3MP8#`K=c4sjq+I$x{ zwuo+RFa5<|g3^&dEyDIgy@Qv6KFynG`KGkc2p0fVS3jzi2)l`H!ItvrXRJp+tcos6C42}%NC-VXipt3Zc zrBu)x*O7VDpHG+~T3fh2&Uvf0uh3~okSVgUn37!lIjZHkSq!(GDmYp90daLqe}I4s zZ&-ckntiBbJjOm=Sw`beR@1TVSo}Or^%xk)8{&c3aX(Gi!@Ij@|tNxSaP9_+0Ivjruto8^;+F!vt+oDrdE zQAOWlore_t1xK@AKZxS*KV`)xaC$Pe=<5fXH{w;evBs2&zv-(uhv-C1}R1t>ylZEZ=#5aZ!&?Frsa|+I7tgqPa++jK$j%LU@(nt@QEOACsFbl)rwXO`7SfT!&G&gNR$7P_QRgl<| z4t5_z4n1J+b9fA>n!pjkY(XB>?}$`hehmx$I~k_D_qJVh@izhd?BFl&lZ}>6(BUYo zeODN0tlALavxD=kx%C-fam^j=ukF z$&ElWOum^^^P)^#Xx6OF`z7%QyvF;op=|D?AGj%YBFcr4x3x=Z5pPE#ailRQ>iokW z6i}K#Y`|}sUf};J4(KD0rorOgy9i(pSQG zbnBs&2pw_s3aQFv-|7X(e56hhJco}^e2rkHokkp&3Zu2t<$&Og1~c8wie^z5UXzU?+kK;svwVAa>6dt zIif%PtXTksgl;HYze7nlE4Ig%TNh_$2n+4j@o`pl=_lo2WccRJmZe*&IgMy%bw$CG zCkAYrtO)ELGj*Ub3y`r^-UZO|RVv>cU2R$W`{m-UHloALZ_7JoCiH|Rb&Uwo5d(*6 zk)g-2e`e0*#hxIEwff!g3Q83+p#eO?Wg?y*qr)}vZ5A@~15GAhHu9)X{`_#=X2ie5 z2gHuJoqahp3i>z*a-XAK`2Fah9I!T_mU*}<1G>y;@`FD>`ZiyoeDhPl`wnW8K|N0< z9;PaJ9qE*1^6S;d)+}!)mOX`C(&;~bGf4mlnA<7B9z|Btfn3qfxy^Eef1+y zy8gJ6jDGEpl{>8+{s3i-BtRe-XABagWcsdvJ=Ln)3jc$8v=~1h{;sa3v16-y+HPD4 zPxOwg-wRs|%KBt1?Hr9y^3h+Eu7~`v_#K!b;GW%j;IaR?s=`9qCLbIg>SF?)<1_sy zn(HEzXj3y!f58|csz&sJYZ@1dg3(&(4vu%VaouXTuv{i^#Oi>+Vi0=6ASM3SkXfg#iR{mXXe&@OwSszzf75XHCyOeNS~*S%SHulh3--# zj4K47)zxG{K}hdpa>Bc&V^Qy*X`s{5W(A4Cjds+fbuK&d;CWM9-F$LcyD-6+%y*r25w`HfT6{}gwd9DzZZbFS2zXiYol@4aT-`~qG$nU5QFdIZ<3Ji7xatWf+TZ9 z7m$j$xFNDC`oMv(AU5Tpv3%ObT?gALLZ5(%R_F>K_M|pzLtUv}1o?3*R<7M5E6ME| zW{$rm>%E;}!~e0tcK4<2w-*-pL1wQ07wUj*Qc%#R%jtT4X^fl>CD;uqaEzjjss&0i z4dDR!B$Bj&-zYEEz}cytzxm4JHK=}q!1~^SyV{Xrh{pIGO=}TocXhEKRE={U%Q^B! zM!l;OkRjlwBQ6ZB7~rZv?~*_#SnfRFZTMJQSo1}2_Hpd>;C(MhnH8w@{GITH28tAQ z*=-hdAo^^l-d);jo_D{mIK|}K|6GmNwvh*eA9})TGR!ky{x{RrCUtKq!}h~xl-1Rb zs2Iqr9%^{cbdPmZ&lbM{25m&9(cdSZh27>tH}4m84~Ixb=DUvdBsCt;RP(!)pm5J& z5~X=_nX`{AI_r+*7Le8ZztcLdi=e)5c7+b`DPs)oZXJ)?$|VW2TIM1=y8)}rXl1uH z1SS|B8C)%1Y%aecAySnT57Ggdb$LgyYOdDkvMQ1!_^mAzgbp{RGq9HY1zU?}l>x#H zNE#6)2Hc8pDqcBn+COkpFiQT|%jK-U;Mbf%$B)OcwAf%Lq{1$R(@BfdVD7%U#qo#8 zR6O*jFo6iHyt_MypBt8W&?AB3j<9d-zbdfAH)B{RHvXT49pk~|c+d!OeO|aCkH^)g zk#Bvg3V*)N)YlVRdK_zb75pOqwN^iUsffJ6j!PT2@8TCtt{O~_6K%P4YME}xMTLAM z9wBC~sE=c~F6+j@0a_2wYrt*&``uO;Fe8VVBR+Sa@tRXB-)X(0saEq5Gx28nftF+i z@C~9!8*ZRuywC!>cK)MFhn#3fN9+cHHLj5y&_& z{dax}IX)_mNP1x>*Qx#uI!XoK-#Xsw+AxFAGN*b0_!As<8=_ZvbNbEzn&6xInAi;|_)2H1@FlK%WFeVzjj@Nv-frZAx>hb&lwy_M<@3V|j4?U13xH+BFINfPnu+r}& zL`-C1ANsLGC|<3GR?*C2?9*Y=!eX@`X`GEb_I*sqS)#~yt~pCY|779`aAC9=I#EIS z6v)miu{mmNL)~DMLw_%MmF_GEAg*ZZMoPci#sVnsy@Tjo;%#52 z$XR7e8qntCS84XH-;*{F8RfO=F9FP=-_0%`qL&76o`17NDDz7xuzy3OH=9MA#A58U zI=C_XZ&H@ijj7aa_p0tfZnpZHzXfdDT9w+R^}5YB-d=JHrMhD<@Z%z{Oyp|#1y^V^ zkKOVA7TynLZ(2g=LLkti?fMN7? zObnnWo^9@>S3g}T4%fZNpF2lI_sU>3^m(M%qBdk>Ut|d^nq%hQby^nw&-f@T9tC4u z6y%rSWX*62MtvSqiCCw^22b`Uj~Mie_<{%l8ar@hfeYvJso3Xppjw)W%?8>tY13}p z%ZsOg`TlPnaemY8M9B7g*$SUI?2Izb>KV9Kbt-6(3m9mqe#GE$LS!-pwt#LkEN@wc zWs(kQtOpu3MoTZ>lCsZ2Uz+y*TC31%b9OLZn-<oLO-r z*+~CByKJT=kUM@mG|Tn#wWA%)&6UWu@EdcPt1yHY%g8KrBT zS$@ahHSRFp2Y%CgsT`@V;tX%9EvE*kYVgx*-hA(2gvvzOOa$(_$7S}=n=dK?$D3a8 z&Ix}q)DhW28c`=vb`-gB?b+O(vTVbG8#@M4&dtb5q5EXoZu^?5`6sqRUF+fB>1Ti3 z{Vu*d^=19hom1VRq4KsYA_D)~^o7R#5$eKL4(rO)Nw-3fIElbmoeTeKHBGis!T%NY z+<{R4?;Ax~CrZjHgsei&ULh+o&MYG{XCxz{vc3o*vyUw+ce33Hao5Mp7KhFj85t!b z9i6Q%ys-{yZKcpg(b4R3^Gi^nV^HN4tIK(36_oAO1ondW zG*v?b`^Jlyg?vrgzG7}zZEsA>ey5urzn8<_5KTM98vUE_&<~xPJMZ>35~H*u-;pDz z60?ojoL+><$2*X&Jthk9@_L)7#;FIc(J?5~Ea9PXkPlrJAuabq_^7*HmmXT9o);e^ zmEa<@YrqgNv*nd*2%(G$N8dMb54Wi*p~=kK5Gy>GX=(jtv%7j+XSHw z2^TL%9zlD5lT6TD+xuJ*s|++V<;s#RMKNT87W&m=W|R&zdL(#X_6mQfk8;r8y>da)bbS%Www|3*?_C|` zN69U#xfpVZylQSWS9W;8n0AN*KQ*T~sNVVsm*P|wxqNrf&4-%3M%FOxHQ!{7Tm$MP zx9qvGGN(YE*W|74!C>4o+U7dltsgolqVzzCMFajGV{YHPP+E0%1Y5gk)elVi$Lip> zdM$zSkdDNlrQ6M9Sv0SniBo;#fj~52dS(k_fpw(gVPmwWJ@ibcty+w2{U7Gm+8Y9w zxQ5jiXO}uHImxq#{ChHJc9GLtLg1V}ygTJbua6H(N3dT0N$4*$Q<*$iT6$_=3hv(@ zhAFWIf2YkF5n;^cBZKMxNtD#q7Ux=rXz5BR7#X~0uR#>Joc%g+m2v|_0O?we7>nq>7p zTBHCL+tBb;@@`!pJeVrWhha}FBur6m7iO;aM2|%wd)<(0o(>!Df z3G#?{M;sLP!QUcA#=QJlk{(jqUW3?mwr)9{;(DJTuNy=?iTAFjBWu;SDY962!4nz# zX45ry5O8dxi0Y6%)0m3CSB%eveRqDZ@F+2$pR^%?tJ;7X2`uNQ`COHPLfLE+l+o2E5=m7?EuS2#w2qQrl5)12g%EDf6xhLcxxD0RG5EXsmZs*Vvqu7yL-SU3>axx_OEwA7s3-!xVo-mn(00;^GiWdP7Ib49O{C95tEBB{p5B+o=7 z&%=Z(ughebz64#SL93efj@IlOwFng9;}&`6UDAwMm=KGGo0aY2^F-na5vbe2=##Yd z(x;Q}LWvw^rOlo?4xcA$E4MzE#}HlzcM#F4Z5t=+8F~uiB35MKRJHKJX$a^^WcXyD z>;`h0m4!1UWLk1N=fU8BdvIHp>B{eiCDR?7&C%PI@qBVAqHaO-utWc%3hHysJEaZ8 z@~+?@?A$@zhli>A_uR_+v=~>MzWt1isSBRUNLj`t3-LML)I^vNLzCt=Ix-Xuu5P`I z6I;o5$P^5}C}FI$;4H$K!F)iYVxR;VQwG`FQhacxMxSe`fKe0}on4wN{HHcftbV>n z+n15n|Fs5JAD~ZJ=}BK{m(5iCBzg;_?!&eEz}S{{h-yI%r5aIDMn{Q?`G^|p?+Bx6 zW~HBL{VCIt!Lam?M&<|46-h4a%j{&w;&+_=sFUKQ^j_^muDBI9z4{B42xibQhxV2} zEgSqm!Ho_^Bm68*w_hu7)5myOjvn?~*~x5pP=;>a*>d%}@dneB|D{|$#^Imrbg)yh z+Nw93)AIgzf_LQ;tv(CHhHXqJShVr5YFP7%OqThc30 zd~mt&?5cS;6}x=z1Sm5GLsE-oCT?Y!;@bvw@Gx_aDkEj+W^nh6v9qhE(P(NWnrd%+ zYxhGQ9#d2N#iit8ZHTGAVZoda8~y*h?b4nMZaxaqaRv1uTJeny+7tGRx0uA&*xEef zve%@<^K)|UC+hryDFD6`Fh3NS*5U@QCnkrTt3Y|Mq7q-oqRY=67P7>+YMmh$u)M@_ z!?9U5z5bEY!b3^ytP$`_&wqCvBD_L`=Zc@6nTl3R=r=s3Y~HTt1pnx{ET#Ohin#E= z`t*yBQsLo8xq}7LM~Wy~5c|C1>-Cj0nb3KSrrCxK`*kR6YzgHUP1YuI+G)}#SBiV6 zJ68?Xvy#7?{Egj^1x$iTCM`?>>!@Or!G#@?3cFBY0itV1Bbvk8@T$Q|iU~FPD!^{A zJ}DUpM{nL854cYmTp@F%WkR1#oFq`;&PIC-*_kIpj7_>fqr?ZRg+(mODFS!ee@PQ4 z01JNAaF-LcHCuDXy)2ARuNnN81fG-hZcf}AII`+AH0IBXtx$*WinJxJ6@1Bgx z2Tkz&{!U(n9%(?W^f8R67ARK-0=<$`l#EOW=jXnUwm~WO6={~%7w7KbrmFvabCIM? zBdgdBV)|;}fpz8>>Afl2I#CYtJ?8dmHd&`94K@{pyX>dt8SV7Wjg{b0px_m5trFRVnSg&K}^A&*ar5 zxLxU8Ivxb9H8ApFTy<_|>vqL8lLJBX7B=Hedqv-=xGz5ssmK#)an6;|&l0S;v2iNS z!7J9}S?tn1@6$|VJ?0O zR4a8>r>I=vm|*7Y4SdO36>O5}2{u{2p!R{|v|Yj4PEPbONJ7~=ZeM5WiC2mn6JPL{ z27xG!jlqH%86LgJu^ljJ_Nv-Zx~!}@5JFZvMow!N<=K@6WWpN`C zRhj(pHgl@;31@u1gp|1b?!S1|tOl*d*Lw)T-vxxvUavvl8{q(?700abos z$sx)fy-OBoM#l2eem_9cdkH@=_F$b0*H`!NWj<**d7?}xlULRQdy>(%asX(XSJA^{ zx6H=P{fDg{*^CO6uNlR>s>h#!Sj-NT6+iAf^A1Ss17%eGO_W*j{*PX7`nrVv7N|qC z{~v<4QIMvvkwde4Tx4HsR|QNpHhgr-OM+k*4^NY_QiSqY5!M#d-7@dT##wH995>?K zd>8oy%L908-{#haVRxqlh{)34_6s44Ya95l6d*iTJ8HMhz8I%gq}AIDkfReX<}QWr zOEqXbdQ>MQe*+rmAD9CVvYLHWWpMpiBM}qu6OIi$Zm}T>0`vFh;M~Gw*ldPsQVm4$ z<%vHLyLZ6F;RF=0Mx|N+G>at`g?61+Ux~J1|5QUm^wG;xWUfpcD~$+mWD;%55qn81 z70T`h8)1Qqz3!y`% z{@+V$5^m?49rw9<<@-k$D1HHCQw) z#^_NhbgB1TyZBAc0@Eu@Z>BS#+bnKe;e-ej6Wf-AexkoNK$olZoyrxX6auNb!ifWC``YYxpy`|SyW^2L2NWS+SI4Hu zrdh-eKuV~rc5V|B(c^lC3fi7{uU^gYVeHGaoY5VRHM_oB-K@_5UFOeA!$zl@th5Ge zq`iEjgzz%n1FZ^2rnxbS17{W{7m_(ytm-NsA0eRCf)LGOc|Tmd8)m(&4+rlq)H_$8+{>xD2=9_*urdY zXus^Oc!ahf6`&4#F(d^fb+HkKSe3%+h<1v76`sx^>MmB*c&@;%iit21|s;1aPxx*OijavmI(2=bV3^k{dy-o z1*IPajcKVl5oJqLOu3ztSj4+2GhDM}hKb_zg$wUVY{f;k@|aIwcP*M)N6$+en(NAr(M37`v;N{o_oI{{pwBMmDPdS;nmd z4pKQKo#g+x9GNS!>r#_sKaID`GRSgvzxuQCWt@$SU{)GYn77*di1f5Qr2EEM5zY8_$hl$f4q%Fq}M^ZBJ`0zd!S-RAL z`VSdj5QeBB(*san9HZ}F3b;w?B&%|$IR4EWUh~>W4aLRbmSq*nYiFXY=SmN(Wn@#tTN)%Hk|z_zUqN2q0+p#gTs9 z^-{l%O^nET(Nu@O_ZGi^Jo*3sndQTIZVgqZ#IVnF8ZTUNAyQly5+-z%{{WtF`dg?zby|A z4lKyY+aBDeYsPvaO$;ov9501z z$`Y%A0_x<80-lldmUpU(n&4#*XS5cXJX0tgUPk+`Io0H=nUsV!DN7tQ_+uY@fR>_RC>a@>r3OQ66@bY@>q_|86)XC-$=0Rox@ zpU3gLXhCpHp?#BsYN(LyGl%hG7+l`Xd`6?e5!m$trJLdL@_lKj3zDo3-W%A^yan8WW( zt}8<{IJz#L3b8GTbHW?3rc=3g5fyImDu;(Y8T+XQ=MHSfN7b&sSc1y6F9dkT%q#P8 z;m~Rr&o?)AL~Od>gb!{ru4V#qkQ7AX&2VN2w$Nr85@+D)E@(2>FBagDcmr;6Viwa& zN}}^BVO=R^*w%A~=*$3?_cjs5R&i`qE1sUwr7)c0+wR7sIT)?SxC@c?#0J#rame1t z<{#HD%)GMn|4g!mxt%0%Cplb&j3*!z-Gi^SHJ2X}E*(B&FnbCyB`SnV*TyU=@>V4J z#7`8ZTB1BpNGvD{E~0!Vu`ga9W^aBGTePKd`;@?@W*ddR{bQFuZQk_3kTJIiD@U^rzS;`>O$0VIQBqi`5fqg*j!lA(;^f~uJ;^r1^ zb=y0OvU1Kg(xhK&EFAu%Y!IM}c)gc&hf3im;#BQmUwGQdt`89J1Aog&53`+y-UI!2 z1HTlLSv!3QlV(m^@-uNW*s)NRU&=Cv%mUyOL@g7I*A&ISCO)^zf+RdH9=JuSsK`#4 ZVS-2RybfLo9isq4N7F!~{H9&l{{faddLsY; literal 0 HcmV?d00001 diff --git a/crates/fjall/logo.png b/crates/fjall/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..8693656da19136596b6bbae8c36b14b212e85719 GIT binary patch literal 17908 zcmYLxby$?o_x7`NcY_EkA<{@HO1Olilys+bN(zezEYgC6q;v>~C>=|Kgramy2qMx7 zNWHW8`TpKNyzqKXiWiI|8W2qL|&jL?Q4Ebw0}hyV}#Iq)0( z3;qzgD;s)2kZLOCA8f1JjSW0x^j0wN)^W4*_OzDa!&OgmW{|VRm;cQu@W9bLKy6|5Xs#sihe;FF{W;U$KE4wzZdt2=&yuOm#p}9-$^u<17ua1qxS!Qy zC)7bp0kZSZVgPhoAb5`s`c-y=Gp{F5fV#sgU3|>;{cK!r7rm>XJarlGqY@G1mu{Ig zSOJuXV?#utr7XoM$~aOWJ~r|?7Ur8wJ6@K#qXQ{(Db?1wJZi$1pU;}-69+MoMI_~fA?3`m8N*eeO2O9 zILzXDvP8$#M6^Bbj^fxDMPlG!irDGd5}ko-)27&zlAFLX=@qdt+xoHVY^@*iuN}B7 z)_$4J*Cr@F2-)Cx@cnGl2>HD+5@y0YVo;~d zbI$Ijc0fb4EK}H+wkml8q9LM|SdEQ3dxh2_+_f9~O_A8dsZNSd#qKLG;wqfa$GJZa{F|Ck=8uAR^C7;4tgMtt~{D_s{B7XM}zLSeKy3L4(aOIsJEdB?k+cd~akbg2 zdsieJoiqy{`@I(ONZPPUJ}=nsIXZECdje}Vx4nW`;qUd5JMhq&iSYKK$;Dm4cMiQ7 z;IX%+_we;~=%5x%qYAck0zED*4MjCOG_s(-iQYbQ(T$(SOKO4{f8*j=eVr-1OS`C~ zqOCuGE&Ez1IN#O8iveByEy14xzK7Ss^_h{}OYDRz*=12JF(2ZG~p9(<{$TLY*%E@FqwU$v6}3&{gP!(?6(9hFoc>WWGc%UsUUTntGA*)-@(X zar%fJGB(}~LaYdeDN^)$vDCV92;k|;Jyzyn?WgCuJNDe(DFEuD?`h*;Z=HU=28UnV zPs57ux>q)^li4ay!4-JDgaV14@UcD>e07ExYUYV<7CxV#Sc-pj#DZ9{rsa7nI34$V zCAMtjV=Wrl&O@um(Y3u6owR_UrXP^tg$P=Nb`Oz9b6j`sQXXgG?6^xmlVqL{9pcG~ zhP`~TM}6GURrk4o;&ZaT#&FuvJDQ_kcGBXMln~26?Ki%_35hXh!%q2dm?}@`LO(rs z>wHY4>yhR|39gG&=_(CpI6C)LFeiH%(ps!}%z>%{t( zkYh9Z2NgXcl64c3bpouA3VQCllO>T(%1~cw&sKGzG=yy9M2w0?YO||(Xf&>xee_~N zvy1X{pT%x4)c??bVTo`}sr`e3YDM$d;CZVD^jOkWW{D1ZozCg)1B^D2340F!{QU?! z5I?5=BYOoCLT`5lLQYZZiv?MAq#&%(O&D`Kn zQ(J;77#n>ev1fxo{7n3S!x|H5x&NzZ+1o=k7rUQ)C3ZJ{{6xrEgd(w95)zEp6g}=lN`W#2=~H|Gwl61k>Hi5AQhAtT?QhV1}?@ zp;az#;qW}Mp%gLcM51zmuyJ=y)R_1PB=-93CxuQSygBn zi9sAx_hxWbp2U_%q1I{d0iT`R?ETQ_Er92JgD{9aon@+ZdhHDpB!A_#X6Vd0s)X=G zQ3Q4rg&Uhdinv%zs${_O1BqvKI2RXVcP(E+LacP!LGMOrDJj!y(Pz~{AH~-a=C*9& zu1RWehH7Hpd;Rims}=M)db$cWIhI_8>YE@dl1DuR8YUi26XKO@G-kx7p zd@g6hxWjTEGDejAdChxqw!s6nQ(vG;Jn&-K;%eNQ*QTh}#~V05mpe73c9esHp#q>+1V6LKnO}n9$YF60REj zJxn#v5+->J`RNbv!r`B!Qd~L>!}iaEMIVx|*$N*?CE^4J{19#zyhyI4t2;M_Hb2WD zGDHl?7NXC@C0`$4T?HDGv^WfqMoMCz?+!eK|*`H#U#88#s&pCG< z>Fs19%d(CJ^TtJ+Y#93W{0n6g7i!jE^%WD z-Ri(lRMAM~na6sEASoJlMM<`qu$}=bNOsdUCZlYK#o2k#oE6ymNI}7P@lE@bEq=f? z$bmisYg5mJgtB2UQH`Ke!EE-CPw}2KaYv{beoW+t7|T{X6G$21!@!H{E;?{*YGhpS zhW+8{^~6pbV%*J z+9+($x|+-7nJ6>h>4>WfSz1W|KV(DO?8}593QEZ0k`HB^J#fk#ya|=`K(dI>PFPvH zLg3K=qMWH{V3w+c3L8rNG^L$;lTLAXDUX)1eW3hi>W0+3= z@dbfKxP3Lp=iXr?e=m@UT1s^>)PO;J?W{kv(Nj(U&si9Bd$r_#=vnn+1{OQcNEJgS zE>#nfNfizK)Y0`qrqaqF@{)m2Jk%U%N-k;*iq zUXO{CI=$a2VMxZ-(~TtU+~!COyEN4AQ|kzNACM1N0-;zo|D)IxreE^w+TZKxe9R-?Pv{D25paB!|7bItXMv8?V?1XZ@G!DeIrPcMV_iRpXyjed z4W<)IQK&5pK?a9Ovyih7zZ^!C-|Fs0X7rSkB!*ow-i$1Yiqa~IXZDEG$F6%`qzdn~o;%8F0$hWaBrc9B71=r><8ryMrXX?Xd|Bt2B*}@uj)mOF_hT z@rVAlhS(fmS8GE`HO&tw#ilVtRmaf>SdnAm8_pdIX%geI%4dCnYRv0`LM9VoFk}g? z8sVmO&DFf3Tj|74j|0vygDpM~iSZXdVRtO6FSUGCRpYbl`@E#TaZk?sC$Q{MqGHcX zJOe>ID39wE25@N~n&Kt8kub?=e5jzthP0AsTyk%YkVPlGK=51N&#gjJ*8R2sB480N zw)@={;aZ5I_%K}>#7vJ}i4B@l|3}SG$;xS1i6Z79P8fOLvR^}4gV73TG^b3MK{M=s zi7lB770=^gf;@CLvK%PUk>V>|y$qe=)OUqQWieH(mf7H@8$dEVsWc*XqgB=CeLRic zkMT@Ms~#dq-7lflg+fYYdQ`}4$qM}vxQxpeZz*KwBQRahyrfN%dN_Dn{eRd>AvJpZ zA|kWfM`{2FHO*#BM(uO9R4$_;G`^Y>S1`~5@U5{x`}iK_uw z13;>Zr2p7VDrw@(M!v3+TJU(me@64%#02Hk8^MZ-TTA~g%5|=78$uCCu6}ibu%Y8|kls?4z6Fb*Ty54el=?)5 z0c4oZ#XqMz*x!Btm!HDJds5E;<+xm*8XL=0OTpfQyEoo@LM3ADm|U#Bw^L6G@}8ziDX<#Y=|itvEc^#AdI z85u!Wc$uaG5y{4RI;#!DW-bY=b>pkuRmA&X!N2QBHZ#ip{kXNK>-$r)tduT;R(kvD z_76f%&0n~w|A((|qqd|*8ATY7zUJVStiAN0oeV-ZHB=r=9}NRzC4rd4NPNxS|3X0x z5o#UNAc>Gk@9TQy90kY~lcQi>vz0EbA^nvuZMm;M*LV~*l>Tz1ALw3PfH zg*0yu%!js`172(c8dAxc!o107xSybgdt>N(C^5J!5vw@;_jx7ca+6m?hw?L5g+ zKAbDc1Gg|0$iV|Xp(K?_W;xhx>|E4qFEoJ$hLK(r-q2_Dl{E)7?pDCy{yT#f*iQ_I zUTdbao7_2L9+htOKR!`2W<;#TabELeC(LbHoOhxFBg%gnb!VgW{Pko=g#Uo`s%*vh zsm^nv2CNsm-d}SfQRCxTp;wLaS`{0Qnw}|-IyES;; z%O7H)`<%Y>DszTn{vX?I2&&5hLuiW?i-l$8n*A?UyS`IhXDm#)%T38&#k)5A2s$+L z>8Flcn>KmU_)s9LvmrDwMW!m5N@rP`Gog>LM-|4QemJKwr!VZ^g9(m5K+F;!6y-n2 zlq7A!-9H~lPfv5|e}xS<%EI~ltAjt31S)uDM(xMO>giH*Re8~Sf{})JLYR4TR*YzIo>!=0 zT<3>4;^UUfpCt8FA<@sLXXif?2iBFjZ-O)z4&o6A(X?3tI368^ymm138?*I&!7JWG zeWTauv8klmZKRNC(RA32#oC@N^*lQc9WBx4Y@wl*ax_?(HIl%wJ}>8K zL3og3)N`788BkMIy$w3+a>p5IsNWE&gu5_*rDnlp^L*;sLRzFMz9s}c_x|Y{=UYpvD z=x;Zd%LYn_m0+mM*MQ&t*rC>7gi3hG3gy$v>JKLd-n?4B?ZNcralhw$^mltYEgpT4 zCa(nA8e5>hIpxd-hI9392;!JdP&)DOaQ-!u{#6fF!E(ZR&>djJwahPBE$2}%B$H!x z*|!lJ10Ti=k-Ic zqN(_HNZ|`DMQ4H6my5Bwc)1t^gp05JZhsMQ{iDO9KXSN~%RPTbGj=yw7S0W-$Z@}r z8}sF2QXek?&R)>ZFgBU$j9W9`EX&gu3x&RV)eg{*iSd6la%p zd+-CRwLN!&fm*3R{_CQidW@-Se9!WKVgu!>_ZxvC^p@+Nr00dg0j}~sjHD9uhDbxP zOPxgPk=EF*&ADdjQ#M3^^sf3b&6W90E3q+8+!n#Ch-3i`A>NWP$F!(HP~1c<4q_bQ zy=6RiXPWO0UFrvAo{^I;jJhhcV}+taqw6u;oq*GkFvumd)V5Bx=(}le*xW_2&b0tw zv{>1ce4%x?b!Z-M^?H3H8xLV-2};QAh8S9*TZhRIbud!K6bc8sYWpzq3yXc^$5@mI z{j;Q=WnfbR3dR`nJ#fFR+ovprcYS{ygNq{^M2#3VFGOcB6tRBET&i=E#cgG4M*&yzUN&&16)9f3E;mAWWSe zW@`j|aj!-0+IpJx7e>gz6mXzyU;mmM1Yh_rd&uC4dA|t%pgMnvKv>@)pZ~WmU2Y2~ zSL`2&SK>D-)h{(rTp+yo2Vb)`&t+s>_fUFsXHFuX0VF>Viav~SLSi2Qrex;w zRs}Pm=ky;tmMu4kA4u6$I_57C6ULA$>85^Ts6Li_u6Q3~EWRM0$o0d7V16e|oyA)j zn6vhcl#!4V0VNTzRt+6CVDKN9Yhzl&S4`Bf(M$cNY&X_%I+}-RC+OnHpg(EcJ*V&Q zSYQ0bCC(tLkr4>RrBnpObI`GtzKZN4Y~0)UTP_5B9?+nFD)}u}{3WxN$VZ0trREDG zPsXg<*HA1LB_%w}9=2d_U3QknPxRZCLdciek&`8;j8?~or;_5El`MP}u)+LtMjy~h zw*Y+xGUNgf`yq5gD5cE{K>VuK)kq4h3^B@>D6DILDCF1=EqYO*J9mYaikd=b^N2kV zep^pWogY~TJO#QaO{u|Aq~*i&KGe$*|79TltR|v~{){%33eOBkrZ^Zg)KNKgX&?f) zcYqxQIp@)UJtJSizco(c0frU%)zE)u0pboj-+0c4T)Xk|2A#e20m)!ZSVsbnH9SSPd)r;4UnVjU>OK5-z0YaTq`UnKbjRkA@Fpws5x=1Yg|~ z3;kyGvvO#wmH5HjUt&2WmU|71kB1Z&qQ1UW8$OZvc!?Re*i{qtCx31>A;3;^^U_B@9b7>xoOKy#~zzZIm5__24z98hJoc(3h% zn`*H`veN<=xa^)%gt@mDUR6wiiL$|@Y21{iSBHXQo`!xc9xfB;toc|dLbnD$9JrZT zO9f{%Z0?Ctu8-6ULd*K~Ejt+?;r1@fps28sMl*wHwUP)vcJK=QfVdL$$0~s-S&h$O zxR%(3y)x&h<4@vw6Z-$^nOG8zQkioN|U^+WG;m+*e{G9+xHqgBpm@R-NlW)u+Y;G-KEjWY|p8qxB%c_?W zLO}mL?l9v}Q_4-qZxCc>Nj!%VC+Rt6Xy<3!@_IZ>oLzL(mrAgE>}m_`B;@gfTrF~S zPr@~z68Y6M^H4uF8WuGgos5mr%kV0NJT9SUOc!urRXD=)0gOR!qPircw z=s_RzWnVh#ufBlJP@O+0dQ`ylup#@VPp=br0bD_{PkZ^PrDUt-$CJgkfG)vC{BAS; ztFuA1L954(az2djyPR#Zn}kp_yfbJEY77OgH@W0;PX~(6Uaxs){<;E6BBt#C06m1c z5U|({c+#Y2WAh3m3N*~uahTqL^aPE&RD{kaJOpR~{I4l1`)i_#jV@p(PDT8m ztK5P`!4$uNIh;EL-m42P5&7(chnvOQKj-QggTbc9B5lW%mUs|;5+(_yl$g7)ejcM{ z?IB*&NaBcGcV_ty>F6pP)23El2^}sNN^f3b*a`iat&S8)7OR#io*0VXMJ>KYm?F{9 zG-Hzx4J|&PE(E2)`jC+vf3SSto7;*9V?OQLs}z7zI>Lim@E#cQqamg!?_5_^n}U{| zV8a9$&QXh(Dzw1;L1p*&6-a(FCP6y7DD)4v^1q)egd|Q&VhEe#PGbBGo3ELNhmFo$@oG29K`1wm&cM=P2D`22w__s@=%21TTIAU4 z4VHyxFJmWE=5*$MP>b~xW=nKrZgCTWB)hCIyWF7eA+BjQxoy7G+$K2qi7!vP3=^Oy zhMC$))tPTXlCmHL?d}k|^R^@)Nb>1A%i+mM_ zn=#sJ_Lp3N_Qx+mhGpAk9Z>FP10H0)A+VfKoWGjc--1J~M#t<0LBMLp8-PW~g7W2C zneU-?GEh$h7e&U}ZCM#rZjrvieT+AMcx?swJDt_};6a#jX7q=AhbCZNH?rH;(+S6H z6R_2GL05&2+PPJ6n4%JqG$s)RvV)-W*Ng?pFSV_SJ{`rv1We#)k43JU9QM6C*_IdZ z61j@_#HH!(i3K>YISV2^5d9<1XsE~e8L(H~k49L_KTSybpwIGv=V}&7f8NALA2r

1O|08GE6Fs`pv57SpaL2#F#3?|)slAEGe~kiT z8L?%Fm7x=BOB*%d0OiT$>wz8#+j|ZQI@UF1s%(2=#?O7!#co3H0Z$>Y_7Qh*hhz9T z6QV))p(&A+0sPgo?5`Nw>SP))^9oZ7dga_lu;ec&dIA#)aK*YmeDd{{I}K8~0+alj zJfT6ta-%M{v>(i!rDt7dRJGpZJ3RUNcTj=OA|2%11=x^U5Ft7%YOJG1P{y5M$Oqhw-(N1~5h@I{ClklL^kKY8iAL zWNn_+eFPwL*S?S0-rx@@2WOSoTDW#(rB|ZtWL|3HJ`wb2lX1~-77l%CrVGl_RX(dJ zn8e+I;<9w>bESJ-xAy&rmlbmV{JlR_+INtTsl=v&h^N1hF&k3ni2_Uvl{~kwi4Q5_ z^#alcESpvMwlMxc8HP{PfjQXL;)p<^=pXVF+z%P7JTi`~sHoR~3B_8!qWfM9@(Sfb z);oS<2cp<$u5R^ECF*&D1AV|IKlz6^KJ8l_FqGC1d%C zFUKl`6k0K2bK^_NsH$@pb{%wOUbZrJ3wvaKbQ4=P%QFR37Ms6I4Y;zoFa!6^H~LoU zGRpK7V82r>rfs=1w!NSLo()uTPn~Ep{zGR;L&(Tgh=eqTEo-G&=7Yuu?e5dTOTJsd z8)Au*0m~)B9O2Px-sO)T>J^sys9-o6keJ8=vfkrqUBi3t!G7l({SbCkWwV6F5ig_d zL5_5%e~30bbtLh=-K8#y_vG-9zYYGQV2byYXB4-iv4zSr54nBd`G7Sm7q((yqhDQe zuVDYNf|;zJSHCq0YGZPL5%LVUc9AnN9&$_>BJZ9_{G(ixGZ~aQWY9-h;9_kT^bi99 z-(znZCtX&l4-{*1tX`v9Qzf@GhlRz2iGPHjy+;UiLvNp6)~n|o{#Z{qA7nCs;@Nyo zX1;crg@-^HCe^Zn3=}ctvwg*ytB~Z3M2yL;sz#E$azs7tOJznpByZZks@WefLU{W1 zqTYt3SD8FnegE?MzElRNa>oyB#Keo92zz4}Yi8-0;*FHSF@?N>rxIJrnOZ@w*5YILD&c(#p={8tt`=*g4Gw^;N(lmby~tk5QzGDj6N zhJMgh=rTq0&!39N6iYMr&sQ`yweOD_wUR;LG|i{Lf+=!{gtzaP0qNf#Sjhf}J5D>9WR z5eV=58blA7&6#zU&s3csz9217Cw-)PZ<J@geN#Lw9fV>L z*hYIb=nJ#pG!|VPe;aGusqcHun4iT)tzz{yAzyh!HZ?Y=81M(%<&C41lCG~SIM1|# z3>T3N42!Hh99tF3U-jObGWPjAclV+-79y#9!g#e63aT{LjGmt%OTBvf@+*cUPzDVR z+Q&t_(y7K{wQxgQvR){bjiTHUl}>-2{Xuk+SsDN9vR7I(pJ>5jICn;RX7A7WBPN^~ zzjU}GUhBJ8>o46Qi-Fav<@ZA~InBuMt72FjQ2JtN1JA;O9`-h1G%I6J6n^{zw>e-t zz7*&oi2 z`E~iG38Xq|0p9p6`w1)iRxj9FgM`ZI=!)nu2Eqe=@OIVe)sF)<=}_8-DH@7VVIw=& zX^C^BBxB0D{X_B!JsGe3Kwz*SyB%6&)5Z!p>SFjVVmnqyAw>mhYD|$ZwA>$`Ps|j0mFWC;=EYLD2YUuzg?1X-HB!tj>Zp@jNjPWqE!RkZ?ngboVFiq|0VL4#Go1;RxR?Tr9&Y?A5H_%4 zBjv;PZ?&H1^8iIPXmF(~h`-bU(=Qi>uD=j8uHUY<96q6T`8>)1q!kZ?$-;O-W&Av8 zhz++6xu;8x(-puNm{;krkys1+Z#* z5MRdrIbulU|Eh2|0I{%9jS^&GQZ!jJv(2+B0~NQdOyF>dpgtlxvd|(FM$cV?VItvu zlOzQz%FH7H!jS0wLvOMW->2#-D5FB)(yeJ2BXI(6$bBqX@`@+T2XQnn<#~*IHUCHe zt{(lOSZ(srS+7#!bzmMPV=t;X?$(Qi$Nic~5XaR{iSSdnH#+*ju2VK|0K-ru%+uOc zDS=A0Y?vB6U3z$RL-w?hI_>6Nc5HdV7WW)A93#j zLFDrc>*!nYww%)Sw_8;_GqR51?g_$TJGAO%U6e??Z_P4^HyOs5$g-lA^%rWL=FGoV z!s;SiW|>+8ve7SO=f7Zl{U%F7q&rS7%(1qs_b{ispU3t1n&;Cw3Fr3L@oiV+MGyT2 z!o!Ha63|bu=!$izg}x~a-Nx_hm&E!EE%*h$k_dt0r>mv7d)^4U*dBJu_$w_3Gnuek zZWM=%1uhKKiB|YL3<0osy0denk>D3F09Ml?l-TZTmI>^Q()4wWZ}ktDg&2(8!!*N0 zoMlp*tw~!_NGxr(T9zLTy(O#;!Znu`#mboLMX{_Yg|ShWnKjp8It%m zoz)oeHC)wBxx5<>UaDtLuq{|XlWYSiN1N8y`e^(i!bIvR?WOj4+}XqJEnQQUxmpMG zm0#jp4s8L~Eq*-iG0?mBCV-uVqa2Gi0=o>G9_IR4KOV6Y&n^G!9FC~)HFIYNu`|<8~%@yzCcj3juoV%)S4Te5E{MEKEf5nuG z$uB)t3|S(O&B{9_3O>Gl#`2fjGiqY4nKV9lvjSXm5Y;gw!1qY{hSrqCdVmjPo^7|B zKaDc6;dm&!?M&v@^rU!0o26VGl8AJkWie`?)Shk>dP#B9oDhH0XVH}e@;NHcr#u*1 z1-_2(+1NA#Q_n4IA^YQ6YWhc5T;5Xx59#xF#DM4`$PpC35bt51Vhbn+^xgN#)?5g^ zYe*&Jt*Wqj~URAF3}&Y{|riN|}PV^PX_R#ii5ha6Z;OLFwYwT;Ubs)G8^9sxZZ60yNY4 zULS|sIf6Ao;FaHF$td#XZ(L)ApP4qMhW&Ur3e-YHl*l$ZptsMj#ue_$etHU=yF~6* zARVYLxL#`^Thb%-7ng4&!bTI(dD@<~vjgDp?_X?=yGeX&{;2F%nd)mUi)#gcg&@9s z$El-(Yio(TUTrdo6W)LL7C(i2^C=t1r31n9=mRAcJxpNv6Ir}<$To&kmJm*3VJU63 zFi-eC(@m8ZaJQ`rOQq)u$fcxG0_-UYgn4+E5t8H>VJS$qR4$rzE_mP|n3#xk~m-AP+75JSIQck}j$9(&# zfjGhh3fZ}qB>5Q=kYs+O_giy3ME`W1FKVR88yIcM5;hy5n3KK6u}}p|8&2MN zeKA9xZ)ALu6xF8_d|XktsJa~z3ntBE^}!7P(f$VABw$#y13gtw+@Q7mhz)I&a_Ap% zQlJj}3FmJX*ZqzA@wAH~l1thBJyhB!+J0Z`r+Wa~40_`{E$Mgb`9i>&3y8#yW&BL; z#*;lRsztOxfqdhKkyL&*Y@k#E*M3uDPbnth>%~6JW{!s}^(G8C9Zdrg^ufS7O3PsI zTx;N10Mg}S&hht7i#`WO8p*x)M>Y4ZF5n_%j?X0(t!B415mk~>FWYU(N&xpv+0%pk zHcVLJNda3%32<}1ZHsJ??K2OehI*(KPDgYwDKR`UAV{d^;o=H!D*;qk7W+v$fYmk$ zgoF3e`9__~_DdKx$7qg&l({1v5yi1~h;r~D&on~vH*hALg5AVL^S}9OzY*^RzNS-Y zD0*UA6KdiktDl9+T(o1tbsuJR^ipLM_ z+hDc_Rzzx(Cl>~KWLKVKlR)IZ1(u>}K&p`!>ubf|f1Ogv| zxmcf1+9%Ep-Zvvn3q7|Z1x6F|u75=Xkx^3>$T~}@Z1%kr+9@pVo%)mkmQz{_@rkbv z=Y_977qq!;+LeUq9dBcWCv?R!n*h32jn&!#By@b6pU;EXr^QGZ{u3K%R=rcIH0%Y` z$Ta+plXYVav{$brhc1@;g~fldn|bX{dRQlzFWi=O)A(7p6F5J~i-k-^UzL`69uCS7)QPvEQDVWo+~h|0^M0&MIK75D>C;iY0;$Q;?VzRl-M_ zOm*3llX(rtqF7ej?*ayM=;e%=K^V7Ub;2@`NNPZ5jjAt&4`|ipxivOgrdcxjYFrTp zj=f*Ld$gPg126e{*+A>*{mU7jG%-7T4hkoM^bcyJ?%dy6t%m?89-UgRS?-1_AjoRd z$`>aYlz94EM4>pT_30EU4F46*`XwIKcMO!Seb< z2RLr5HOKM5Lk(Kb?*C@oYY*riLN^sZaEbvnt0c(@6_8NpggB=WJEHo0$Vhv%N`Oup zG!(FN-xt2*=M28srR)L}4k4f`>Le-20AjP&6AltmfB+S&17& z==Ny&Y5#Yuqal zSaobiqFp9xmusD7^iMN`raDZDVy2vrSCI*y)w_sk5!wpLWpO_|5C~RoPWN12%AAYz zIRIrwNeLqGy`D)yaMw!TRPCk-BgInq4XW@Pb~UzY#VSH&Vq=_CiIL9a_`98&b7B1K zy*`HAjhPfO^rHoIr97{&9BQxplAHbD#Jn~A!DTAmoQR-Wp7wrAfXDOK&!(IiYkr7$ zZ~fxTy9F`@&CJ+u`oUJI0Cpc1PW?$v5c!fV;dl#~yYV!$$az|st}JaRMc~aH11but z0NVTcM3p)md4+#w-_pj1oVrkuWg2ld_RWrHO>q?@_c2r7gMv+b6kK(xojDE$u;u03 z-a2@Q8+XXpc4Uzt?jFp=w-Wlc1r!6nN#P8wxqbUj^X`|X^P3h>iLqo!%ex1=!{st& zADp`c@t*m$BebhbYLhI5^({}V;p73ifuS%Yi>fRc7JcMdAv!nVi1q`gzox?@S%V0@ zBxU-c=&vNUu09-lC=kEjCh#xoO}uH2j#8#2%)}KKI#C@dETc!};9_G*xR8G<`t@3j z{)T7ywZydh1X#D%9W#1Az5XfL~w_J*p1X49geK55oI$O7mYH$5|!Ne z*oirAHc+BN4)~LjaBN(~#AZijUIIORDY*G}7jn}xrWa$XyG2#HFC(*K6ttr*0bI?2 z(BDz1cRmnVxP62hcTWWJ8yaY#czATAVtj=TrF`3F@u5;)?Cq;!LPEvM{Bi_ zYIZ|8n4W|{`eb3dXK3agu)zQzZ+1@GLF$11nW{7y3>?g00J?3faioer5L}|(4*syA z-M3bNrdGWOo+3bkt6r7u+4vL=K9&gHZx59`o0#jucfvI1V{0LD0ybMd^`&u_nLk~E zXPve%2=caGD#4lmy7vVojxxT|PkO!=R@Hf89+I>n(pT42bNklgHn$&bCSymvbW6yD z7vNxA7^c~+CVD_d7}+-?tNzyUDnTOY{aK+2CeYgtdf}9%T@`VoH}KDY8t?oy1zA`H z7fHEQ+eYKhKi>sUCU~(S%;mOi*pXnxjo7~a`msc&Rr-Y>;`7Zla{yp_jD*MWz#z}- zcrTtzLAgH@eeVwiw{pIuY?yszd}W{3KnxG3(OrJsE2kOowD+Kj5QLaK>i;!!*z7mf zCP)uDLy=Hi=ezl2;N4YY z9%#L%%S~mmNxdQl1-X~!=WI?h_I%T9#+HjU#ww*-^>yg-tu#7h}IcxA95n)8V*}u8Wl$o>$u(nHPEI?WWUCi;$=LcC<=} zpPdzP^Lw(Nq_H8fEW99>!VH6<*ut!GQ%S|!cR4-$>97;i7S zUEr7n>O_JyX`*^*WqTMaF|TLvgZd#U1m!YQ=Om$1na!9erSR%*)y_vQ)Dnjr)C7Wv zlg%0IrqsJtRq>b62^SgNi=zqE7bX@`CI-H#9TTq97s}zyci#J46!5+d*Zn|e8$JBk zm9QYfcyfetKJdrn!pk>|5F{%Y7ow)JOHRrgi;|8szVh?%IAZ6rT-nCv7o=3X3@vztTR|8$hVWI@jpn?Vj;XcBA7XWNUHGd1d zee2KZ_W+rON?=Q!6RH(11haI?2r?ajZl?8PS_F$x3;xVo&2ji|zZ7wZ@oZ1my0}@j zm_N|}=bAr}M{A9$*WlWH{l-^9&aTug5pT~`tTgoVc7@E3R0oEbnrpdTHuWb+xLR^T z9ED<0S$)_W_W+VPJ-D+@fct9iXvIZ)sXA-s&nf7*?ltrP+}Ly=2A9ez8{5)QeTA_A3q6a2A@tyxY@J7%SIN~goQ#u&aUOlTgHyW;CqQ5YHywY7Ce)j_wN-e8{kN# zNC>HJvEHVaIQ^YL2)@JMZ+&`#JW3yuCZME*rSMaZO5(^mQKLtGT?w2gw&>i~nC5aS zd88er3wdP(vB2RA%R@Qij~*%?Q#vnwCwIOJSV<=QnR)~?1qoN* z*tcD%h+lyqi?#8&I}}Gk;^~Sg6reg{u;Y;|)WEL`)JyTc&NC+!5qDR(d|`aN_G&?fP>$0P zHZ>*X@uvx2@6F>mMHvSj@MfZ*S`aZ-+t(O;Za91om%a6a_0KI$EQh0^#@9NKYTuQa zKS5X&A!7x-nBT=i+g!$nt~!{3?@l!2TOU55S|U~tA_HT7qD;65>>a;zn0H?yB-dXD zO2yo|#tm)5Lg!b42|aummYm~w@U(69J;$Y49vMRTkh5DqH5Dag!*ii$Z$o@e##=Ps zuS^-y0%+3D3~J(9l_Q)Tf5*)UhkLd%8uyW#)%duDlxM$9cY-S4dHvjCo%~RTvk?14Y1RB;e2`}E^g|AEwm~xDw-L{z@kCa*ETs-y|&=vJ=giU|&UP^?>?h>H28#<06pHVE}xaV&efuET)iwY^sXB&Jz)#>SDU1xn zb8Fkj9Sb+A;5?z!_&a4}0sonrjCVAcBr{zv;n+Ej~q9IT)Xe?{IVMJaV65qokBVLEoXE zmPyOgoQz#A@M(yj-(EpG&*y7REZ6YEDRQuP$S|EHh7ekOraSsRToD?Aw{^X|vY1tX z1gh$z<(@B^I;tPt$ziWn2VZzVtPtC>cTv}N)%iZ&yo;6k_=ArRRL_x=K3f{TT15~= zj(zlKXVMZ5cYMRd7Ecr!IVi%~y4Bd?KZfg)!i|O24)5qv7gImwW?v0SMr#sUL~p!8 zq557vhk`k9$#9*I*K1t@smq8RIX>Yc2ZmhwnOA+Wr%xb|4ZFc8bWOqiU#SdVU-?OvW=g$DB(UbJYJ0MCK$ MD`+4pz>% literal 0 HcmV?d00001 diff --git a/crates/fjall/renovate.json b/crates/fjall/renovate.json new file mode 100644 index 000000000..39a2b6e9a --- /dev/null +++ b/crates/fjall/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} diff --git a/crates/fjall/src/batch/item.rs b/crates/fjall/src/batch/item.rs new file mode 100644 index 000000000..f0b64f91d --- /dev/null +++ b/crates/fjall/src/batch/item.rs @@ -0,0 +1,73 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::Keyspace; +use lsm_tree::{UserKey, UserValue, ValueType}; + +#[derive(Clone, PartialEq, Eq)] +pub struct Item { + /// Keyspace + pub keyspace: Keyspace, + + /// User-defined key - an arbitrary byte array + /// + /// Supports up to 2^16 bytes + pub key: UserKey, + + /// User-defined value - an arbitrary byte array + /// + /// Supports up to 65535 bytes + pub value: UserValue, + + /// Tombstone marker - if this is true, the value has been deleted + pub value_type: ValueType, +} + +impl std::fmt::Debug for Item { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{:?}:{} => {:?}", + self.keyspace.id, + self.key, + match self.value_type { + ValueType::Value => "V", + ValueType::Tombstone => "T", + ValueType::WeakTombstone => "W", + ValueType::Indirection => "Vb", + }, + self.value + ) + } +} + +impl Item { + pub fn new, V: Into>( + keyspace: Keyspace, + key: K, + value: V, + value_type: ValueType, + ) -> Self { + let k = key.into(); + let v = value.into(); + + assert!(!k.is_empty()); + + assert!( + u16::try_from(k.len()).is_ok(), + "Keys can be up to 65535 bytes long" + ); + assert!( + u32::try_from(v.len()).is_ok(), + "Values can be up to 2^32 bytes long" + ); + + Self { + keyspace, + key: k, + value: v, + value_type, + } + } +} diff --git a/crates/fjall/src/batch/mod.rs b/crates/fjall/src/batch/mod.rs new file mode 100644 index 000000000..9ee49757a --- /dev/null +++ b/crates/fjall/src/batch/mod.rs @@ -0,0 +1,181 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod item; + +use crate::{Database, Keyspace, PersistMode}; +use item::Item; +use lsm_tree::{AbstractTree, UserKey, UserValue, ValueType}; +use std::collections::HashSet; + +/// An atomic write batch +/// +/// Allows atomically writing across keyspaces inside the [`Database`]. +pub struct WriteBatch { + pub(crate) data: Vec, + db: Database, + durability: Option, +} + +impl WriteBatch { + /// Initializes a new write batch. + /// + /// This function is called by [`Database::batch`]. + pub(crate) fn new(db: Database) -> Self { + Self { + data: Vec::new(), + db, + durability: None, + } + } + + /// Initializes a new write batch with preallocated capacity. + /// + /// ### Note + /// + /// "Capacity" refers to the number of batch item slots, not their size in memory. + #[must_use] + pub fn with_capacity(db: Database, capacity: usize) -> Self { + Self { + data: Vec::with_capacity(capacity), + db, + durability: None, + } + } + + /// Gets the number of batched items. + #[must_use] + pub fn len(&self) -> usize { + self.data.len() + } + + /// Returns `true` if there are no batches items (yet). + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Sets the durability level. + #[must_use] + pub fn durability(mut self, mode: Option) -> Self { + self.durability = mode; + self + } + + /// Inserts a key-value pair into the batch. + pub fn insert, V: Into>(&mut self, p: &Keyspace, key: K, value: V) { + self.data + .push(Item::new(p.clone(), key, value, ValueType::Value)); + } + + /// Removes a key-value pair. + pub fn remove>(&mut self, p: &Keyspace, key: K) { + self.data + .push(Item::new(p.clone(), key, vec![], ValueType::Tombstone)); + } + + /// Adds a weak tombstone marker for a key. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// # Experimental + /// + /// This function is currently experimental. + #[doc(hidden)] + pub fn remove_weak>(&mut self, p: &Keyspace, key: K) { + self.data + .push(Item::new(p.clone(), key, vec![], ValueType::WeakTombstone)); + } + + /// Commits the batch to the [`Database`] atomically. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[allow(clippy::missing_panics_doc)] + pub fn commit(mut self) -> crate::Result<()> { + if self.is_empty() { + return Ok(()); + } + + log::trace!("batch: Acquiring journal writer"); + let mut journal_writer = self.db.supervisor.journal.get_writer()?; + + // IMPORTANT: Check the poisoned flag after getting journal mutex, otherwise TOCTOU + if self.db.is_poisoned.is_poisoned() { + return Err(crate::Error::Poisoned); + } + + let batch_seqno = self.db.supervisor.seqno.next(); + + journal_writer.write_batch(self.data.iter(), self.data.len(), batch_seqno)?; + + if let Some(mode) = self.durability { + if let Err(e) = journal_writer.persist(mode) { + self.db.is_poisoned.poison(); + + log::error!( + "persist failed, which is a FATAL, and possibly hardware-related, failure: {e:?}" + ); + + return Err(crate::Error::Poisoned); + } + } + + // TODO: maybe we can use a stack alloc hashset/vec here, such as smallset + #[expect(clippy::mutable_key_type)] + let mut keyspaces_with_possible_stall = HashSet::new(); + + #[expect(clippy::expect_used)] + let keyspaces = self + .db + .supervisor + .keyspaces + .read() + .expect("lock is poisoned"); + + let mut batch_size = 0u64; + + log::trace!("Applying batch (size={}) to memtable(s)", self.data.len()); + + for item in std::mem::take(&mut self.data) { + // TODO: need a better, generic write op + let (item_size, _) = match item.value_type { + ValueType::Value => item.keyspace.tree.insert(item.key, item.value, batch_seqno), + ValueType::Tombstone => item.keyspace.tree.remove(item.key, batch_seqno), + ValueType::WeakTombstone => item.keyspace.tree.remove_weak(item.key, batch_seqno), + ValueType::Indirection => unreachable!(), + }; + + batch_size += item_size; + + // IMPORTANT: Clone the handle, because we don't want to keep the keyspaces lock open + keyspaces_with_possible_stall.insert(item.keyspace.clone()); + } + + self.db.supervisor.snapshot_tracker.publish(batch_seqno); + + drop(journal_writer); + + log::trace!("batch: Freed journal writer"); + + drop(keyspaces); + + // IMPORTANT: Add batch size to current write buffer size + // Otherwise write buffer growth is unbounded when using batches + self.db.supervisor.write_buffer_size.allocate(batch_size); + + // Check each affected keyspace for write stall/halt + for keyspace in &keyspaces_with_possible_stall { + let memtable_size = keyspace.tree.active_memtable().size(); + keyspace.check_memtable_rotate(memtable_size); + keyspace.local_backpressure(); + } + + Ok(()) + } +} diff --git a/crates/fjall/src/builder.rs b/crates/fjall/src/builder.rs new file mode 100644 index 000000000..6fc7a7cfa --- /dev/null +++ b/crates/fjall/src/builder.rs @@ -0,0 +1,190 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{db_config::CompactionFilterAssigner, tx::single_writer::Openable, Config}; +use lsm_tree::{Cache, CompressionType, DescriptorTable}; +use std::{marker::PhantomData, path::Path, sync::Arc}; + +/// Database builder +pub struct Builder { + inner: Config, + _phantom: PhantomData, +} + +impl Builder { + pub(crate) fn new(path: &Path) -> Self { + Self { + inner: Config::new(path), + _phantom: PhantomData, + } + } + + #[doc(hidden)] + #[must_use] + pub fn into_config(self) -> Config { + self.inner + } + + /// Opens the database, creating it if it does not exist. + /// + /// # Errors + /// + /// Errors if an I/O error occurred, or if the database can not be opened. + pub fn open(self) -> crate::Result { + O::open(self.inner) + } + + /// Sets the cache capacity in bytes. + /// + /// It is recommended to configure the block cache capacity to be ~20-25% of the available memory - or more **if** the data set _fully_ fits into memory. + #[must_use] + pub fn cache_size(mut self, size_bytes: u64) -> Self { + self.inner.cache = Arc::new(Cache::with_capacity_bytes(size_bytes)); + self + } + + /// Sets the compression type to use for large values that are written into the journal file. + #[must_use] + pub fn journal_compression(mut self, comp: CompressionType) -> Self { + self.inner.journal_compression_type = comp; + self + } + + /// If `false`, write batches or transactions automatically flush data to the operating system. + /// + /// Default = false + /// + /// Set to `true` to handle persistence manually, e.g. manually using `PersistMode::SyncData` for ACID transactions. + #[must_use] + pub fn manual_journal_persist(mut self, flag: bool) -> Self { + self.inner.manual_journal_persist = flag; + self + } + + /// Sets the number of worker threads. + /// + /// Default = min(# CPU cores, 4) + /// + /// # Panics + /// + /// Panics, if below 1. + #[must_use] + pub fn worker_threads(self, n: usize) -> Self { + #[cfg(not(test))] + assert!(n > 0, "worker count must be at least 1"); + + self.worker_threads_unchecked(n) + } + + #[doc(hidden)] + #[must_use] + pub fn worker_threads_unchecked(mut self, n: usize) -> Self { + self.inner.worker_threads = n; + self + } + + /// Sets the upper limit for cached file descriptors. + /// + /// # Note + /// + /// Setting to None is currently not supported. + /// + /// # Panics + /// + /// Panics if n < 10 or `None`. + #[must_use] + pub fn max_cached_files(mut self, n: Option) -> Self { + self.inner.descriptor_table = n.map(|n| Arc::new(DescriptorTable::new(n))); + self + } + + /// Maximum size of all journals in bytes. + /// + /// Default = 512 MiB + /// + /// # Panics + /// + /// Panics if < 64 MiB. + /// + /// Same as `max_total_wal_size` in `RocksDB`. + #[must_use] + pub fn max_journaling_size(mut self, bytes: u64) -> Self { + assert!(bytes >= 64 * 1_024 * 1_024); + + self.inner.max_journaling_size_in_bytes = bytes; + self + } + + /// Maximum size of all memtables in bytes. + /// + /// Similar to `db_write_buffer_size` in `RocksDB`, however it is disabled by default in `RocksDB`. + /// + /// Set to `u64::MAX` or `0` to disable it. + /// + /// Default = off + /// + /// # Panics + /// + /// Panics if bytes < 1 MiB. + #[doc(hidden)] + #[must_use] + #[deprecated = "todo"] + pub fn max_write_buffer_size(mut self, bytes: Option) -> Self { + if let Some(bytes) = bytes { + assert!(bytes >= 1_024 * 1_024); + } + + self.inner.max_write_buffer_size_in_bytes = bytes; + self + } + + /// Sets the `Database` to clean upon drop. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{PersistMode, Database, KeyspaceCreateOptions}; + /// # let folder = tempfile::tempdir()?.into_path(); + /// let db = Database::builder(&folder).temporary(true).open()?; + /// + /// assert!(folder.try_exists()?); + /// drop(db); + /// assert!(!folder.try_exists()?); + /// # + /// # Ok::<_, fjall::Error>(()) + /// ``` + #[must_use] + pub fn temporary(mut self, flag: bool) -> Self { + self.inner.clean_path_on_drop = flag; + self + } + + /// Installs a factory that assigns compaction filters to new or recovered keyspaces. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{PersistMode, Database, KeyspaceCreateOptions}; + /// # use std::sync::Arc; + /// # let folder = tempfile::tempdir()?.keep(); + /// use lsm_tree::compaction::filter::Factory; + /// + /// let db = Database::builder(&folder) + /// .temporary(true) + /// .with_compaction_filter_factories( + /// Arc::new(|keyspace| { + /// // Match on the keyspace name to assign specific compaction filters + /// todo!() + /// }) + /// ) + /// .open()?; + /// + /// # + /// # Ok::<_, fjall::Error>(()) + /// ``` + pub fn with_compaction_filter_factories(mut self, f: CompactionFilterAssigner) -> Self { + self.inner.compaction_filter_factory_assigner = Some(f); + self + } +} diff --git a/crates/fjall/src/compaction/mod.rs b/crates/fjall/src/compaction/mod.rs new file mode 100644 index 000000000..8b716bbf8 --- /dev/null +++ b/crates/fjall/src/compaction/mod.rs @@ -0,0 +1,17 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub(crate) mod worker; + +pub use lsm_tree::compaction::{Fifo, Leveled, Levelled}; + +/// Compaction filter utilities +pub mod filter { + pub use lsm_tree::compaction::filter::{ + CompactionFilter, Context, Factory, ItemAccessor, Verdict, + }; + + /// Alias for compaction filter return type + pub type CompactionFilterResult = lsm_tree::Result; +} diff --git a/crates/fjall/src/compaction/worker.rs b/crates/fjall/src/compaction/worker.rs new file mode 100644 index 000000000..33b121cfa --- /dev/null +++ b/crates/fjall/src/compaction/worker.rs @@ -0,0 +1,60 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{snapshot_tracker::SnapshotTracker, stats::Stats, Keyspace}; +use lsm_tree::AbstractTree; +use std::time::Instant; + +/// Runs a single run of compaction. +pub fn run( + keyspace: &Keyspace, + snapshot_tracker: &SnapshotTracker, + stats: &Stats, +) -> crate::Result<()> { + use std::sync::atomic::Ordering::Relaxed; + + if keyspace.is_deleted.load(Relaxed) { + return Ok(()); + } + + log::trace!( + "Checking compaction strategy for keyspace {:?}", + keyspace.name, + ); + + let strategy = keyspace.config.compaction_strategy.clone(); + + stats.active_compaction_count.fetch_add(1, Relaxed); + + log::debug!("Compacting keyspace {:?}", keyspace.name); + + let start = Instant::now(); + + if let Err(e) = keyspace + .tree + .compact(strategy.clone(), snapshot_tracker.get_seqno_safe_to_gc()) + { + log::error!("Compaction failed: {e:?}"); + stats.active_compaction_count.fetch_sub(1, Relaxed); + + return Err(e.into()); + } + + // TODO: we need feedback from the compaction strategy... + // TODO: if there is nothing more to do, we should clear the compaction_manager semaphore + + // NOTE: Throttle a bit to avoid a storm of compaction choice attempts + // (in case of write throttling) + std::thread::sleep(std::time::Duration::from_millis(1)); + + #[expect(clippy::cast_possible_truncation)] + stats + .time_compacting + .fetch_add(start.elapsed().as_micros() as u64, Relaxed); + + stats.active_compaction_count.fetch_sub(1, Relaxed); + stats.compactions_completed.fetch_add(1, Relaxed); + + Ok(()) +} diff --git a/crates/fjall/src/db.rs b/crates/fjall/src/db.rs new file mode 100644 index 000000000..ecc1b8814 --- /dev/null +++ b/crates/fjall/src/db.rs @@ -0,0 +1,921 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + batch::WriteBatch, + db_config::Config, + file::{fsync_directory, KEYSPACES_FOLDER, LOCK_FILE, VERSION_MARKER}, + flush::manager::FlushManager, + journal::{manager::JournalManager, writer::PersistMode, Journal}, + keyspace::{name::is_valid_keyspace_name, KeyspaceKey}, + locked_file::LockedFileGuard, + meta_keyspace::MetaKeyspace, + poison::{PoisonDart, PoisonSignal}, + recovery::{recover_keyspaces, recover_sealed_memtables}, + snapshot::Snapshot, + snapshot_tracker::SnapshotTracker, + stats::Stats, + supervisor::{Supervisor, SupervisorInner}, + tx::single_writer::Openable, + version::FormatVersion, + worker_pool::{WorkerMessage, WorkerPool}, + write_buffer_manager::WriteBufferManager, + HashMap, Keyspace, KeyspaceCreateOptions, +}; +use lsm_tree::{AbstractTree, SequenceNumberCounter}; +use std::{ + fs::remove_dir_all, + path::Path, + sync::{atomic::AtomicUsize, Arc, RwLock}, +}; + +pub type Keyspaces = HashMap; + +pub struct DatabaseInner { + pub(crate) meta_keyspace: MetaKeyspace, + + /// Database configuration + #[doc(hidden)] + pub config: Config, + + #[doc(hidden)] + pub supervisor: Supervisor, + + /// Stop signal when database is dropped to stop background threads + pub(crate) stop_signal: lsm_tree::stop_signal::StopSignal, + + /// Counter of background threads + pub(crate) active_thread_counter: Arc, + + /// True if fsync failed + pub(crate) is_poisoned: PoisonSignal, + + pub(crate) stats: Arc, + + pub(crate) keyspace_id_counter: SequenceNumberCounter, + + pub worker_pool: WorkerPool, + + pub(crate) lock_file: LockedFileGuard, +} + +impl Drop for DatabaseInner { + fn drop(&mut self) { + log::debug!("Dropping database"); + + self.stop_signal.send(); + + let _ = self.worker_pool.rx.drain().count(); + + while self + .active_thread_counter + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + { + let _ = self.worker_pool.sender.send(WorkerMessage::Close); + std::thread::sleep(std::time::Duration::from_micros(10)); + } + + // Drain again after threads are closed + let _ = self.worker_pool.rx.drain().count(); + + // IMPORTANT: Break cyclic Arcs + self.supervisor.flush_manager.clear(); + self.supervisor + .keyspaces + .write() + .expect("lock is poisoned") + .clear(); + self.supervisor + .journal_manager + .write() + .expect("lock is poisoned") + .clear(); + + if self.config.clean_path_on_drop { + log::info!( + "Deleting database because temporary=true: {}", + self.config.path.display(), + ); + + if let Err(err) = remove_dir_all(&self.config.path) { + log::warn!( + "Failed to clean up path: {} - {err}", + self.config.path.display() + ); + } + } + + #[cfg(feature = "__internal_whitebox")] + crate::drop::decrement_drop_counter(); + } +} + +/// A database is a single logical database +/// which can house multiple keyspaces +/// +/// In your application, you should create a single database +/// and keep it around for as long as needed +/// (as long as you are using its keyspaces). +#[derive(Clone)] +#[doc(alias = "database")] +#[doc(alias = "collection")] +pub struct Database(pub(crate) Arc); + +impl std::ops::Deref for Database { + type Target = DatabaseInner; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Openable for Database { + fn open(config: Config) -> crate::Result + where + Self: Sized, + { + Self::open(config) + } +} + +impl Database { + /// Opens a cross-keyspace snapshot. + /// + /// # Caution + /// + /// Note that for serializable semantics you need to use a transactional database instead. + #[must_use] + pub fn snapshot(&self) -> Snapshot { + Snapshot::new(self.supervisor.snapshot_tracker.open()) + } + + /// Creates a new database builder to create or open a database at `path`. + pub fn builder(path: impl AsRef) -> crate::DatabaseBuilder { + crate::DatabaseBuilder::new(path.as_ref()) + } + + /// Initializes a new atomic write batch. + /// + /// Items may be written to multiple keyspaces, which + /// will be be updated atomically when the batch is committed. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// let mut batch = db.batch(); + /// + /// assert_eq!(tree.len()?, 0); + /// batch.insert(&tree, "1", "abc"); + /// batch.insert(&tree, "3", "abc"); + /// batch.insert(&tree, "5", "abc"); + /// + /// assert_eq!(tree.len()?, 0); + /// + /// batch.commit()?; + /// assert_eq!(tree.len()?, 3); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn batch(&self) -> WriteBatch { + let mut batch = WriteBatch::new(self.clone()); + + if !self.config.manual_journal_persist { + batch = batch.durability(Some(PersistMode::Buffer)); + } + + batch + } + + // TODO: refactor: accessor to stats(), so we don't have that many methods in DB + + /// Returns the current write buffer size (active + sealed memtables). + /// + /// # Experimental + /// + /// This is a non-stable API currently. + #[must_use] + #[doc(hidden)] + pub fn write_buffer_size(&self) -> u64 { + self.supervisor.write_buffer_size.get() + } + + /// Returns the number of queued memtable flush tasks. + /// + /// # Experimental + /// + /// This is a non-stable API currently. + #[doc(hidden)] + #[must_use] + pub fn outstanding_flushes(&self) -> usize { + self.supervisor.flush_manager.len() + } + + /// Returns the time all compactions took until now. + /// + /// # Experimental + /// + /// This is a non-stable API currently. + #[doc(hidden)] + #[must_use] + pub fn time_compacting(&self) -> std::time::Duration { + let us = self + .stats + .time_compacting + .load(std::sync::atomic::Ordering::Relaxed); + + std::time::Duration::from_micros(us) + } + + /// Returns the number of active compactions currently running. + /// + /// # Experimental + /// + /// This is a non-stable API currently. + #[doc(hidden)] + #[must_use] + pub fn active_compactions(&self) -> usize { + self.stats + .active_compaction_count + .load(std::sync::atomic::Ordering::Relaxed) + } + + /// Returns the number of completed compactions. + /// + /// # Experimental + /// + /// This is a non-stable API currently. + #[doc(hidden)] + #[must_use] + pub fn compactions_completed(&self) -> usize { + self.stats + .compactions_completed + .load(std::sync::atomic::Ordering::Relaxed) + } + + /// Returns the number of journals on disk. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// assert_eq!(1, db.journal_count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn journal_count(&self) -> usize { + self.supervisor + .journal_manager + .read() + .expect("lock is poisoned") + .journal_count() + } + + /// Returns the disk space usage of the journal. + #[doc(hidden)] + pub fn journal_disk_space(&self) -> crate::Result { + Ok(self.supervisor.journal.get_writer()?.len()? + + self + .supervisor + .journal_manager + .read() + .expect("lock is poisoned") + .disk_space_used()) + } + + /// Returns the disk space usage of the entire database. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let _tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert!(db.disk_space()? > 0); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn disk_space(&self) -> crate::Result { + let journal_size = self.journal_disk_space()?; + + let keyspaces_size = self + .supervisor + .keyspaces + .read() + .expect("lock is poisoned") + .values() + .map(Keyspace::disk_space) + .sum::(); + + Ok(journal_size + keyspaces_size) + } + + /// Flushes the active journal. The durability depends on the [`PersistMode`] + /// used. + /// + /// Persisting only affects durability, NOT consistency! Even without flushing + /// data is crash-safe. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{PersistMode, Database, KeyspaceCreateOptions}; + /// # let folder = tempfile::tempdir()?; + /// let db = Database::builder(folder).open()?; + /// let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + /// + /// items.insert("a", "hello")?; + /// + /// db.persist(PersistMode::SyncAll)?; + /// # + /// # Ok::<_, fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn persist(&self, mode: PersistMode) -> crate::Result<()> { + if self.is_poisoned.is_poisoned() { + return Err(crate::Error::Poisoned); + } + + if let Err(e) = self.supervisor.journal.persist(mode) { + self.is_poisoned.poison(); + + log::error!( + "flush failed, which is a FATAL, and possibly hardware-related, failure: {e:?}" + ); + + return Err(crate::Error::Poisoned); + } + + Ok(()) + } + + #[doc(hidden)] + #[must_use] + pub fn cache_capacity(&self) -> u64 { + self.config.cache.capacity() + } + + #[doc(hidden)] + #[must_use] + pub fn cache_size(&self) -> u64 { + self.config.cache.size() + } + + /// Opens a database in the given directory. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn open(config: Config) -> crate::Result { + log::debug!( + "cache capacity={}MiB", + config.cache.capacity() / 1_024 / 1_024, + ); + + let db = Self::create_or_recover(config)?; + // db.start_background_threads()?; + + #[cfg(feature = "__internal_whitebox")] + crate::drop::increment_drop_counter(); + + Ok(db) + } + + /// Same as [`Database::open`], but does not start background threads. + /// + /// Needed to open a database without threads for testing. + /// + /// Should not be user-facing. + #[doc(hidden)] + pub fn create_or_recover(config: Config) -> crate::Result { + if config.path.join(VERSION_MARKER).try_exists()? { + Self::recover(config) + } else { + Self::create_new(config) + } + } + + /// Destroys the keyspace, removing all data associated with it. + /// + /// The keyspace folder will not be deleted until all references to it are dropped, + /// so calling this is safe, even if the keyspace is still accessed in another thread. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[expect(clippy::needless_pass_by_value)] + pub fn delete_keyspace(&self, handle: Keyspace) -> crate::Result<()> { + self.meta_keyspace.remove_keyspace(&handle.name)?; + + handle + .is_deleted + .store(true, std::sync::atomic::Ordering::Release); + + Ok(()) + } + + /// Creates or opens a keyspace. + /// + /// If the keyspace does not yet exist, it will be created configured with `create_options`. + /// Otherwise simply a handle to the existing keyspace will be returned. + /// + /// Keyspace names can be up to 255 characters long and can not be empty. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + /// + /// # Panics + /// + /// Panics if the keyspace name is invalid. + pub fn keyspace( + &self, + name: &str, + create_options: impl FnOnce() -> KeyspaceCreateOptions, + ) -> crate::Result { + assert!(is_valid_keyspace_name(name)); + + let keyspaces = self.supervisor.keyspaces.write().expect("lock is poisoned"); + + Ok(if let Some(keyspace) = keyspaces.get(name) { + keyspace.clone() + } else { + let name: KeyspaceKey = name.into(); + + let keyspace_id = self.keyspace_id_counter.next(); + + let mut opts = create_options(); + + // Install compaction filter factory if needed + if let Some(f) = self + .config + .compaction_filter_factory_assigner + .as_ref() + .and_then(|f| f(&name)) + { + opts = opts.with_compaction_filter_factory(f); + } + + let handle = Keyspace::create_new(keyspace_id, self, name.clone(), opts)?; + + self.meta_keyspace + .create_keyspace(keyspace_id, &name, handle.clone(), keyspaces)?; + + #[cfg(feature = "__internal_whitebox")] + crate::drop::increment_drop_counter(); + + handle + }) + } + + /// Returns the number of keyspaces. + #[must_use] + pub fn keyspace_count(&self) -> usize { + self.supervisor + .keyspaces + .read() + .expect("lock is poisoned") + .len() + } + + /// Gets a list of all keyspace names in the database. + #[must_use] + pub fn list_keyspace_names(&self) -> Vec { + self.supervisor + .keyspaces + .read() + .expect("lock is poisoned") + .keys() + .cloned() + .collect() + } + + /// Returns `true` if the keyspace with the given name exists. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// assert!(!db.keyspace_exists("default")); + /// db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert!(db.keyspace_exists("default")); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn keyspace_exists(&self, name: &str) -> bool { + self.meta_keyspace.keyspace_exists(name) + } + + /// Gets the would-be-next sequence number. + #[must_use] + #[doc(hidden)] + pub fn seqno(&self) -> crate::SeqNo { + self.supervisor.seqno.get() + } + + /// Gets the currently visible sequence number. + #[must_use] + #[doc(hidden)] + pub fn visible_seqno(&self) -> crate::SeqNo { + self.supervisor.snapshot_tracker.get() + } + + fn check_version>(path: P) -> crate::Result<()> { + let bytes = std::fs::read(path.as_ref().join(VERSION_MARKER))?; + + if let Some(version) = FormatVersion::parse_file_header(&bytes) { + if version == FormatVersion::V2 { + log::error!( + "It looks like you are trying to open a V2 database - the database needs a manual migration, a tool is available at https://github.com/fjall-rs/migrate-v2-v3." + ); + } + if version as u8 > 4 { + log::error!( + "It looks like you are trying to open a database from the future. Are you a time traveller?" + ); + } + if version != FormatVersion::V4 { + return Err(crate::Error::InvalidVersion(Some(version))); + } + } else { + return Err(crate::Error::InvalidVersion(None)); + } + + Ok(()) + } + + /// Recovers existing database from directory. + #[expect(clippy::too_many_lines)] + #[doc(hidden)] + pub fn recover(config: Config) -> crate::Result { + log::info!("Recovering database at {}", config.path.display()); + + // Check version + Self::check_version(&config.path)?; + + let lock_file = LockedFileGuard::try_acquire(&config.path.join(LOCK_FILE))?; + + // TODO: + // let recovery_mode = config.journal_recovery_mode; + + // Reload active journal + let journal_recovery = Journal::recover( + &config.path, + config.journal_compression_type, + config.journal_compression_threshold, + )?; + log::debug!("journal recovery result: {journal_recovery:#?}"); + + let active_journal = Arc::new(journal_recovery.active); + active_journal.get_writer()?.persist(PersistMode::SyncAll)?; + + let sealed_journals = journal_recovery.sealed; + + let journal_manager = JournalManager::new(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let meta_tree = lsm_tree::Config::new( + config.path.join(KEYSPACES_FOLDER).join("0"), + seqno.clone(), + visible_seqno.clone(), + ) + .use_cache(config.cache.clone()) + .use_descriptor_table(config.descriptor_table.clone()) + .expect_point_read_hits(true) + .data_block_size_policy(crate::config::BlockSizePolicy::all(4_096)) + .data_block_hash_ratio_policy(crate::config::HashRatioPolicy::all(8.0)) + .data_block_compression_policy(crate::config::CompressionPolicy::disabled()) + .data_block_restart_interval_policy(crate::config::RestartIntervalPolicy::all(1)) + .index_block_compression_policy(crate::config::CompressionPolicy::disabled()) + .filter_policy(crate::config::FilterPolicy::new([ + lsm_tree::config::FilterPolicyEntry::Bloom( + lsm_tree::config::BloomConstructionPolicy::FalsePositiveRate(0.0001), + ), + lsm_tree::config::FilterPolicyEntry::Bloom( + lsm_tree::config::BloomConstructionPolicy::FalsePositiveRate(0.01), + ), + ])) + .open()?; + + let keyspaces = Arc::new(RwLock::default()); + + let meta_keyspace = MetaKeyspace::new( + meta_tree, + keyspaces.clone(), + seqno.clone(), + visible_seqno.clone(), + ); + + let supervisor = Supervisor::new(SupervisorInner { + db_config: config.clone(), + keyspaces, + flush_manager: FlushManager::new(), + write_buffer_size: WriteBufferManager::default(), + snapshot_tracker: SnapshotTracker::new(visible_seqno), + journal: active_journal, + journal_manager: Arc::new(RwLock::new(journal_manager)), + seqno, + }); + + let active_thread_counter = Arc::::default(); + let stats = Arc::::default(); + + // Construct (empty) database, then fill back with keyspace data + let inner = DatabaseInner { + supervisor, + worker_pool: WorkerPool::prepare(), + keyspace_id_counter: SequenceNumberCounter::new(1), + meta_keyspace: meta_keyspace.clone(), + config, + stop_signal: lsm_tree::stop_signal::StopSignal::default(), + active_thread_counter, + is_poisoned: PoisonSignal::default(), + stats, + lock_file, + }; + + let db = Self(Arc::new(inner)); + + // Recover keyspaces + recover_keyspaces(&db, &meta_keyspace)?; + + // Recover sealed memtables by walking through old journals + recover_sealed_memtables( + &db, + &sealed_journals + .into_iter() + .map(|(_, x)| x) + .collect::>(), + )?; + + { + #[expect(clippy::expect_used)] + let keyspaces = db.supervisor.keyspaces.read().expect("lock is poisoned"); + + // NOTE: If this triggers, the last sealed memtable + // was not correctly rotated + for keyspace in keyspaces.values() { + if keyspace.tree.active_memtable().size() > 0 { + log::error!( + "Active memtable is not empty after recovery for keyspace {:?} - recovery failed", + keyspace.name + ); + return Err(crate::Error::Unrecoverable); + } + } + + // NOTE: We only need to recover the active journal, if it actually existed before + // nothing to recover, if we just created it + if !journal_recovery.was_active_created { + log::trace!("Recovering active memtables from active journal"); + + let reader = db.supervisor.journal.get_reader()?; + + for batch in reader { + let batch = batch?; + + for item in batch.items { + let Some(keyspace_name) = db.meta_keyspace.resolve_id(item.keyspace_id)? + else { + continue; + }; + + let Some(keyspace) = keyspaces.get(&keyspace_name) else { + continue; + }; + + let tree = &keyspace.tree; + + match item.value_type { + lsm_tree::ValueType::Value => { + tree.insert(item.key, item.value, batch.seqno); + } + lsm_tree::ValueType::Tombstone => { + tree.remove(item.key, batch.seqno); + } + lsm_tree::ValueType::WeakTombstone => { + tree.remove_weak(item.key, batch.seqno); + } + lsm_tree::ValueType::Indirection => { + unreachable!() + } + } + } + + for keyspace_id in &batch.cleared_keyspaces { + let Some(keyspace_name) = db.meta_keyspace.resolve_id(*keyspace_id)? else { + continue; + }; + + let Some(keyspace) = keyspaces.get(&keyspace_name) else { + continue; + }; + + keyspace.tree.clear().ok(); + } + } + + for keyspace in keyspaces.values() { + let size = keyspace.tree.active_memtable().size(); + + log::trace!( + "Recovered active memtable of size {size}B for keyspace {:?} ({} items)", + keyspace.name, + keyspace.tree.active_memtable().len(), + ); + + // IMPORTANT: Add active memtable size to current write buffer size + db.supervisor.write_buffer_size.allocate(size); + + // Recover seqno + let maybe_next_seqno = keyspace + .tree + .get_highest_seqno() + .map(|x| x + 1) + .unwrap_or_default(); + + db.supervisor.seqno.fetch_max(maybe_next_seqno); + log::debug!("Database seqno is now {}", db.supervisor.seqno.get()); + } + } + } + + db.supervisor + .snapshot_tracker + .set(db.supervisor.seqno.get()); + + db.supervisor.snapshot_tracker.gc(); + + for keyspace in db + .supervisor + .keyspaces + .read() + .expect("lock is poisoned") + .values() + { + if keyspace.tree.sealed_memtable_count() > 0 { + log::debug!( + "Queuing keyspace {:?} to get flushed because sealed memtables > 0", + keyspace.name(), + ); + + // IMPORTANT: Add task to flush manager, so it can be flushed + db.supervisor + .flush_manager + .enqueue(Arc::new(crate::flush::Task { + keyspace: keyspace.clone(), + })); + + keyspace.worker_messager.send(WorkerMessage::Flush).ok(); + } else if keyspace.tree.l0_run_count() > 0 { + log::debug!( + "Queuing keyspace {:?} to maybe get compacted because L0 runs > 0", + keyspace.name(), + ); + + keyspace + .worker_messager + .send(WorkerMessage::Compact(keyspace.clone())) + .ok(); + } + } + + db.worker_pool.start( + db.config.worker_threads, + &db.supervisor, + &db.stats, + &PoisonDart::new(db.is_poisoned.clone()), + &db.active_thread_counter, + )?; + + log::trace!("Database recovery successful"); + + Ok(db) + } + + #[doc(hidden)] + pub fn create_new(config: Config) -> crate::Result { + log::info!("Creating database at {}", config.path.display()); + + std::fs::create_dir_all(&config.path)?; + + let lock_file = LockedFileGuard::create_new(&config.path.join(LOCK_FILE))?; + + let journal_folder_path = &config.path; + let keyspaces_folder_path = config.path.join(KEYSPACES_FOLDER); + + std::fs::create_dir_all(&keyspaces_folder_path)?; + + let active_journal_path = journal_folder_path.join("0.jnl"); + let journal = Journal::create_new(&active_journal_path)?.with_compression( + config.journal_compression_type, + config.journal_compression_threshold, + ); + let journal = Arc::new(journal); + + // NOTE: Lastly, fsync version marker, which contains the version + let mut marker = std::fs::File::create_new(config.path.join(VERSION_MARKER))?; + FormatVersion::V4.write_file_header(&mut marker)?; + marker.sync_all()?; + + // IMPORTANT: fsync folders on Unix + fsync_directory(&keyspaces_folder_path)?; + fsync_directory(&config.path)?; + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let meta_tree = lsm_tree::Config::new( + config.path.join(KEYSPACES_FOLDER).join("0"), + seqno.clone(), + visible_seqno.clone(), + ) + .use_cache(config.cache.clone()) + .use_descriptor_table(config.descriptor_table.clone()) + .expect_point_read_hits(true) + .data_block_size_policy(crate::config::BlockSizePolicy::all(4_096)) + .data_block_hash_ratio_policy(crate::config::HashRatioPolicy::all(8.0)) + .data_block_compression_policy(crate::config::CompressionPolicy::disabled()) + .data_block_restart_interval_policy(crate::config::RestartIntervalPolicy::all(1)) + .index_block_compression_policy(crate::config::CompressionPolicy::disabled()) + .filter_policy(crate::config::FilterPolicy::new([ + lsm_tree::config::FilterPolicyEntry::Bloom( + lsm_tree::config::BloomConstructionPolicy::FalsePositiveRate(0.0001), + ), + lsm_tree::config::FilterPolicyEntry::Bloom( + lsm_tree::config::BloomConstructionPolicy::FalsePositiveRate(0.01), + ), + ])) + .open()?; + + let keyspaces = Arc::new(RwLock::default()); + + let meta_keyspace = MetaKeyspace::new( + meta_tree, + keyspaces.clone(), + seqno.clone(), + visible_seqno.clone(), + ); + + let supervisor = Supervisor::new(SupervisorInner { + db_config: config.clone(), + keyspaces, + flush_manager: FlushManager::new(), + write_buffer_size: WriteBufferManager::default(), + snapshot_tracker: SnapshotTracker::new(visible_seqno), + journal, + journal_manager: Arc::new(RwLock::new(JournalManager::new())), + seqno, + }); + + let active_thread_counter = Arc::::default(); + let stats = Arc::::default(); + + let inner = DatabaseInner { + supervisor, + worker_pool: WorkerPool::prepare(), + keyspace_id_counter: SequenceNumberCounter::new(1), + meta_keyspace, + config, + stop_signal: lsm_tree::stop_signal::StopSignal::default(), + active_thread_counter, + is_poisoned: PoisonSignal::default(), + stats, + lock_file, + }; + + let db = Self(Arc::new(inner)); + + db.worker_pool.start( + db.config.worker_threads, + &db.supervisor, + &db.stats, + &PoisonDart::new(db.is_poisoned.clone()), + &db.active_thread_counter, + )?; + + Ok(db) + } +} diff --git a/crates/fjall/src/db_config.rs b/crates/fjall/src/db_config.rs new file mode 100644 index 000000000..0ea722201 --- /dev/null +++ b/crates/fjall/src/db_config.rs @@ -0,0 +1,95 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::path::absolute_path; +use lsm_tree::{Cache, CompressionType, DescriptorTable}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +pub type CompactionFilterAssigner = + Arc Option> + Send + Sync>; + +/// Global database configuration +#[derive(Clone)] +pub struct Config { + /// Base path of database + pub(crate) path: PathBuf, + + /// When true, the path will be deleted upon drop + pub(crate) clean_path_on_drop: bool, + + #[doc(hidden)] + pub cache: Arc, + + /// Descriptor table that will be shared between keyspaces + pub(crate) descriptor_table: Option>, + + /// Max size of all journals in bytes + pub(crate) max_journaling_size_in_bytes: u64, // TODO: should be configurable during runtime: AtomicU64 + + /// Max size of all active memtables + /// + /// This can be used to cap the memory usage if there are + /// many (possibly inactive) keyspaces. + pub(crate) max_write_buffer_size_in_bytes: Option, // TODO: should be configurable during runtime: AtomicU64 + + pub(crate) manual_journal_persist: bool, + + /// Number of concurrent worker threads + pub(crate) worker_threads: usize, + + pub(crate) journal_compression_type: CompressionType, + + pub(crate) journal_compression_threshold: usize, + + // pub(crate) journal_recovery_mode: RecoveryMode, + // + pub(crate) compaction_filter_factory_assigner: Option, +} + +const DEFAULT_CPU_CORES: usize = 4; + +fn get_open_file_limit() -> usize { + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + return 900; + + #[cfg(target_os = "windows")] + return 400; + + #[cfg(target_os = "macos")] + return 150; +} + +impl Config { + /// Creates a new configuration + pub fn new(path: &Path) -> Self { + let queried_cores = std::thread::available_parallelism().map(usize::from); + let worker_threads = queried_cores.unwrap_or(1).min(DEFAULT_CPU_CORES); + + Self { + path: absolute_path(path), + clean_path_on_drop: false, + descriptor_table: Some(Arc::new(DescriptorTable::new(get_open_file_limit()))), + max_write_buffer_size_in_bytes: None, + max_journaling_size_in_bytes: /* 512 MiB */ 512 * 1_024 * 1_024, + worker_threads, + // journal_recovery_mode: RecoveryMode::default(), + manual_journal_persist: false, + + #[cfg(not(feature = "lz4"))] + journal_compression_type: CompressionType::None, + + #[cfg(feature = "lz4")] + journal_compression_type: CompressionType::Lz4, + + journal_compression_threshold: 4_096, + + cache: Arc::new(Cache::with_capacity_bytes(/* 32 MiB */ 32 * 1_024 * 1_024)), + + compaction_filter_factory_assigner: None, + } + } +} diff --git a/crates/fjall/src/db_test.rs b/crates/fjall/src/db_test.rs new file mode 100644 index 000000000..794111983 --- /dev/null +++ b/crates/fjall/src/db_test.rs @@ -0,0 +1,244 @@ +use crate::{Database, KeyspaceCreateOptions, KvSeparationOptions}; +use test_log::test; + +#[test_log::test] +fn clear_recover_sealed() -> crate::Result<()> { + use crate::{Database, KeyspaceCreateOptions}; + + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert!(tree.is_empty()?); + + tree.insert("a", "a")?; + assert!(tree.contains_key("a")?); + + tree.clear()?; + assert!(tree.is_empty()?); + + tree.rotate_memtable_and_wait()?; + assert!(tree.is_empty()?); + db.supervisor.journal.get_writer()?.rotate()?; + + tree.insert("b", "a")?; + assert!(tree.contains_key("b")?); + } + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert!(!tree.contains_key("a")?); + assert!(tree.contains_key("b")?); + } + + Ok(()) +} + +// TODO: investigate: flaky on macOS??? +#[cfg(feature = "__internal_whitebox")] +#[test] +#[ignore = "restore"] +fn whitebox_db_drop() -> crate::Result<()> { + use crate::Database; + + { + let folder = tempfile::tempdir()?; + + assert_eq!(0, crate::drop::load_drop_counter()); + let db = Database::builder(&folder).open()?; + assert_eq!(5, crate::drop::load_drop_counter()); + + drop(db); + assert_eq!(0, crate::drop::load_drop_counter()); + } + + { + let folder = tempfile::tempdir()?; + + assert_eq!(0, crate::drop::load_drop_counter()); + let db = Database::builder(&folder).open()?; + assert_eq!(5, crate::drop::load_drop_counter()); + + let tree = db.keyspace("default", Default::default)?; + assert_eq!(6, crate::drop::load_drop_counter()); + + drop(tree); + drop(db); + assert_eq!(0, crate::drop::load_drop_counter()); + } + + { + let folder = tempfile::tempdir()?; + + assert_eq!(0, crate::drop::load_drop_counter()); + let db = Database::builder(&folder).open()?; + assert_eq!(5, crate::drop::load_drop_counter()); + + let _tree = db.keyspace("default", Default::default)?; + assert_eq!(6, crate::drop::load_drop_counter()); + + let _tree2 = db.keyspace("different", Default::default)?; + assert_eq!(7, crate::drop::load_drop_counter()); + } + + assert_eq!(0, crate::drop::load_drop_counter()); + + Ok(()) +} + +#[cfg(feature = "__internal_whitebox")] +#[test] +#[ignore = "restore"] +fn whitebox_db_drop_2() -> crate::Result<()> { + use crate::{Database, KeyspaceCreateOptions}; + + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("tree", KeyspaceCreateOptions::default)?; + let tree2 = db.keyspace("tree1", KeyspaceCreateOptions::default)?; + + tree.insert("a", "a")?; + tree2.insert("b", "b")?; + + tree.rotate_memtable_and_wait()?; + } + + assert_eq!(0, crate::drop::load_drop_counter()); + + Ok(()) +} + +#[test] +pub fn test_exotic_keyspace_names() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + let db = Database::builder(&folder).open()?; + + for name in ["hello$world", "hello#world", "hello.world", "hello_world"] { + let tree = db.keyspace(name, KeyspaceCreateOptions::default)?; + tree.insert("a", "a")?; + assert_eq!(1, tree.len()?); + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn recover_sealed_smoke_test() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + for i in 0_u128..3 { + let db = Database::create_or_recover(Database::builder(folder.path()).into_config())?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert_eq!(i, tree.len()?.try_into().unwrap()); + + tree.insert(i.to_be_bytes(), i.to_be_bytes())?; + assert_eq!(i + 1, tree.len()?.try_into().unwrap()); + + tree.rotate_memtable_and_wait()?; + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn recover_sealed_order() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(folder.path()) + .worker_threads_unchecked(0) + .open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + tree.insert("a", "a")?; + tree.rotate_memtable()?; + + tree.insert("a", "b")?; + tree.rotate_memtable()?; + + tree.insert("a", "c")?; + tree.rotate_memtable()?; + } + + { + let db = Database::create_or_recover(Database::builder(folder.path()).into_config())?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert_eq!(b"c", &*tree.get("a")?.unwrap()); + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn recover_sealed_blob() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + for i in 0_u128..3 { + let db = Database::create_or_recover(Database::builder(folder.path()).into_config())?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default() + .max_memtable_size(1_000) + .with_kv_separation(Some(KvSeparationOptions::default())) + })?; + + assert_eq!(i, tree.len()?.try_into().unwrap()); + + tree.insert(i.to_be_bytes(), i.to_be_bytes().repeat(1_024))?; + assert_eq!(i + 1, tree.len()?.try_into().unwrap()); + + tree.rotate_memtable_and_wait()?; + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn recover_sealed_pair_1() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + for i in 0_u128..3 { + let db = Database::create_or_recover(Database::builder(folder.path()).into_config())?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().max_memtable_size(1_000) + })?; + let tree2 = db.keyspace("default2", || { + KeyspaceCreateOptions::default() + .max_memtable_size(1_000) + .with_kv_separation(Some(KvSeparationOptions::default())) + })?; + + assert_eq!(i, tree.len()?.try_into().unwrap()); + assert_eq!(i, tree2.len()?.try_into().unwrap()); + + let mut batch = db.batch(); + batch.insert(&tree, i.to_be_bytes(), i.to_be_bytes()); + batch.insert(&tree2, i.to_be_bytes(), i.to_be_bytes().repeat(1_024)); + batch.commit()?; + + assert_eq!(i + 1, tree.len()?.try_into().unwrap()); + assert_eq!(i + 1, tree2.len()?.try_into().unwrap()); + + tree.rotate_memtable_and_wait()?; + } + + Ok(()) +} diff --git a/crates/fjall/src/drop.rs b/crates/fjall/src/drop.rs new file mode 100644 index 000000000..d5473cf94 --- /dev/null +++ b/crates/fjall/src/drop.rs @@ -0,0 +1,25 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::sync::atomic::Ordering::Relaxed; +use std::sync::{atomic::AtomicUsize, OnceLock}; + +static DROP_COUNTER: OnceLock = OnceLock::new(); + +pub fn increment_drop_counter() { + get_drop_counter().fetch_add(1, Relaxed); +} + +pub fn decrement_drop_counter() { + get_drop_counter().fetch_sub(1, Relaxed); +} + +pub fn get_drop_counter<'a>() -> &'a AtomicUsize { + DROP_COUNTER.get_or_init(AtomicUsize::default) +} + +#[must_use] +pub fn load_drop_counter() -> usize { + get_drop_counter().load(Relaxed) +} diff --git a/crates/fjall/src/error.rs b/crates/fjall/src/error.rs new file mode 100644 index 000000000..b99a2c45e --- /dev/null +++ b/crates/fjall/src/error.rs @@ -0,0 +1,83 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + journal::error::RecoveryError as JournalRecoveryError, version::FormatVersion, CompressionType, +}; + +/// Errors that may occur in the storage engine +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// Error inside LSM-tree + Storage(lsm_tree::Error), + + /// I/O error + Io(std::io::Error), + + /// Error during journal recovery + JournalRecovery(JournalRecoveryError), + + /// Invalid or unparsable data format version + InvalidVersion(Option), + + /// Decompression failed + Decompress(CompressionType), + + /// Invalid journal trailer detected + InvalidTrailer, + + /// Invalid tag detected during decoding + InvalidTag((&'static str, u8)), + + /// A previous flush / commit operation failed, indicating a hardware-related failure + /// + /// Future writes will not be accepted as consistency cannot be guaranteed. + /// + /// **At this point, it's best to let the application crash and try to recover.** + /// + /// More info: + Poisoned, + + /// Keyspace is deleted + KeyspaceDeleted, + + /// Database is locked. + Locked, + + /// Database is unrecoverable, see logs for details + Unrecoverable, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FjallError: {self:?}") + } +} + +impl From for Error { + fn from(inner: std::io::Error) -> Self { + Self::Io(inner) + } +} + +impl From for Error { + fn from(inner: lsm_tree::Error) -> Self { + Self::Storage(inner) + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Storage(inner) => Some(inner), + Self::Io(inner) => Some(inner), + Self::JournalRecovery(inner) => Some(inner), + _ => None, + } + } +} + +/// Result helper type +pub type Result = std::result::Result; diff --git a/crates/fjall/src/file.rs b/crates/fjall/src/file.rs new file mode 100644 index 000000000..0a6ccbccb --- /dev/null +++ b/crates/fjall/src/file.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::path::Path; + +pub const MAGIC_BYTES: &[u8] = &[b'F', b'J', b'L', 3]; + +pub const KEYSPACES_FOLDER: &str = "keyspaces"; + +pub const LOCK_FILE: &str = "lock"; +pub const VERSION_MARKER: &str = "version"; + +pub const LSM_CURRENT_VERSION_MARKER: &str = "current"; + +#[cfg(not(target_os = "windows"))] +pub fn fsync_directory>(path: P) -> std::io::Result<()> { + let path = path.as_ref(); + + let file = std::fs::File::open(path).inspect_err(|e| { + log::error!("Failed to open directory at {}: {e:?}", path.display()); + })?; + + debug_assert!(file.metadata()?.is_dir()); + + file.sync_all().inspect_err(|e| { + log::error!("Failed to fsync directory at {}: {e:?}", path.display()); + }) +} + +#[cfg(target_os = "windows")] +pub fn fsync_directory>(_path: P) -> std::io::Result<()> { + // Cannot fsync directory on Windows + Ok(()) +} diff --git a/crates/fjall/src/flush/manager.rs b/crates/fjall/src/flush/manager.rs new file mode 100644 index 000000000..fb621e047 --- /dev/null +++ b/crates/fjall/src/flush/manager.rs @@ -0,0 +1,44 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::flush::Task; +use std::sync::Arc; + +pub struct FlushManager { + sender: flume::Sender>, + receiver: flume::Receiver>, +} + +impl FlushManager { + pub fn new() -> Self { + let (tx, rx) = flume::bounded(1_000); + + Self { + sender: tx, + receiver: rx, + } + } + + pub fn len(&self) -> usize { + self.receiver.len() + } + + pub fn clear(&self) { + let _ = self.receiver.drain().count(); + } + + pub fn wait_for_empty(&self) { + while !self.receiver.is_empty() { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + + pub fn enqueue(&self, task: Arc) { + self.sender.send(task).ok(); + } + + pub fn dequeue(&self) -> Option> { + self.receiver.try_recv().ok() + } +} diff --git a/crates/fjall/src/flush/mod.rs b/crates/fjall/src/flush/mod.rs new file mode 100644 index 000000000..3bbe1bb4f --- /dev/null +++ b/crates/fjall/src/flush/mod.rs @@ -0,0 +1,9 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod manager; +pub mod task; +pub mod worker; + +pub use task::Task; diff --git a/crates/fjall/src/flush/task.rs b/crates/fjall/src/flush/task.rs new file mode 100644 index 000000000..80d6b0e80 --- /dev/null +++ b/crates/fjall/src/flush/task.rs @@ -0,0 +1,15 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::Keyspace; + +pub struct Task { + pub(crate) keyspace: Keyspace, +} + +impl std::fmt::Debug for Task { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FlushTask({})", self.keyspace.name) + } +} diff --git a/crates/fjall/src/flush/worker.rs b/crates/fjall/src/flush/worker.rs new file mode 100644 index 000000000..71557c079 --- /dev/null +++ b/crates/fjall/src/flush/worker.rs @@ -0,0 +1,42 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + flush::Task, snapshot_tracker::SnapshotTracker, stats::Stats, + write_buffer_manager::WriteBufferManager, +}; +use lsm_tree::AbstractTree; + +/// Runs flush logic. +pub fn run( + task: &Task, + write_buffer_manager: &WriteBufferManager, + snapshot_tracker: &SnapshotTracker, + _stats: &Stats, +) -> crate::Result<()> { + log::debug!("Flushing keyspace {:?}", task.keyspace.name); + + let gc_watermark = snapshot_tracker.get_seqno_safe_to_gc(); + + let flush_lock = task.keyspace.tree.get_flush_lock(); + + match task + .keyspace + .tree + .flush(&flush_lock, gc_watermark) + .inspect_err(|e| { + log::error!("Flush error: {e:?}"); + })? { + Some(flushed_bytes) => { + write_buffer_manager.free(flushed_bytes); + + log::debug!("Flush completed"); + } + None => { + log::trace!("Flush did not return a table"); + } + } + + Ok(()) +} diff --git a/crates/fjall/src/guard.rs b/crates/fjall/src/guard.rs new file mode 100644 index 000000000..d2287b3c5 --- /dev/null +++ b/crates/fjall/src/guard.rs @@ -0,0 +1,153 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use lsm_tree::{Guard as _Guard, UserKey, UserValue}; + +/// Guard to access key-value pairs +pub struct Guard(pub(crate) lsm_tree::IterGuardImpl); + +impl Guard { + // TODO: is_ok? + + /// Accesses the key-value pair if the predicate returns `true`. + /// + /// The predicate receives the key - if returning `false`, the value + /// may not be loaded if the tree is key-value separated. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("abc", "my_value")?; + /// + /// let (k,v) = tree.prefix("a") + /// .next() + /// .unwrap() + /// .into_inner_if(|key| key.starts_with(b"a"))?; + /// + /// assert_eq!(b"abc", &*k); + /// assert_eq!(Some(b"my_value".as_slice()), v.as_deref()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn into_inner_if( + self, + pred: impl Fn(&crate::UserKey) -> bool, + ) -> crate::Result<(UserKey, Option)> { + self.0.into_inner_if(pred).map_err(Into::into) + } + + /// Returns the key-value tuple. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let (k,v) = tree.prefix("a") + /// .next() + /// .unwrap() + /// .into_inner()?; + /// + /// assert_eq!(b"a", &*k); + /// assert_eq!(b"my_value", &*v); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn into_inner(self) -> crate::Result { + self.0.into_inner().map_err(Into::into) + } + + /// Returns the key. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let item = tree.prefix("a").next().unwrap().key()?; + /// assert_eq!(b"a", &*item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn key(self) -> crate::Result { + self.0.key().map_err(Into::into) + } + + /// Returns the value size. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let item = tree.prefix("a").next().unwrap().size()?; + /// assert_eq!(8, item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn size(self) -> crate::Result { + self.0.size().map_err(Into::into) + } + + /// Returns the value. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let item = tree.prefix("a").next().unwrap().value()?; + /// assert_eq!(b"my_value", &*item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn value(self) -> crate::Result { + self.0.value().map_err(Into::into) + } +} diff --git a/crates/fjall/src/ingestion.rs b/crates/fjall/src/ingestion.rs new file mode 100644 index 000000000..6e658514a --- /dev/null +++ b/crates/fjall/src/ingestion.rs @@ -0,0 +1,100 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{worker_pool::WorkerMessage, Keyspace}; +use lsm_tree::{AnyIngestion, UserKey, UserValue}; + +pub struct Ingestion<'a> { + keyspace: &'a Keyspace, + inner: AnyIngestion<'a>, +} + +impl<'a> Ingestion<'a> { + pub fn new(keyspace: &'a Keyspace) -> crate::Result { + let inner = keyspace.tree.ingestion()?; + Ok(Self { keyspace, inner }) + } + + pub fn write, V: Into>( + &mut self, + key: K, + value: V, + ) -> crate::Result<()> { + self.inner.write(key, value).map_err(Into::into) + } + + #[doc(hidden)] + pub fn write_prevalidated, V: Into>( + &mut self, + key: K, + value: V, + ) -> crate::Result<()> { + self.inner + .write_prevalidated(key, value) + .map_err(Into::into) + } + + pub fn write_tombstone>(&mut self, key: K) -> crate::Result<()> { + self.inner.write_tombstone(key).map_err(Into::into) + } + + #[doc(hidden)] + pub fn write_weak_tombstone>(&mut self, key: K) -> crate::Result<()> { + self.inner.write_weak_tombstone(key).map_err(Into::into) + } + + #[doc(hidden)] + pub fn write_prevalidated_weak_tombstone>( + &mut self, + key: K, + ) -> crate::Result<()> { + self.inner + .write_prevalidated_weak_tombstone(key) + .map_err(Into::into) + } + + pub fn finish(self) -> crate::Result<()> { + // NOTE: We hold to avoid a race condition with concurrent writes: + // + // write ingest + // lock journal + // | + // next seqno=1 + // | + // --------------finish + // flush + // seqno=2 + // register + // | + // ----------------- + // | + // insert seqno=1 + let _journal_lock = self.keyspace.supervisor.journal.get_writer(); + + self.finish_inner() + } + + /// Finishes the ingestion without taking the global journal writer lock. + /// + /// The caller must ensure that no journaled writes can run concurrently in + /// the same database. Exclusive ingestions into independent keyspaces may + /// still finish concurrently. + pub fn finish_exclusive(self) -> crate::Result<()> { + self.finish_inner() + } + + fn finish_inner(self) -> crate::Result<()> { + self.inner + .finish() + .inspect(|()| { + self.keyspace + .worker_messager + .try_send(WorkerMessage::Compact(self.keyspace.clone())) + .ok(); + + self.keyspace.supervisor.snapshot_tracker.gc(); + }) + .map_err(Into::into) + } +} diff --git a/crates/fjall/src/iter.rs b/crates/fjall/src/iter.rs new file mode 100644 index 000000000..7d61102b5 --- /dev/null +++ b/crates/fjall/src/iter.rs @@ -0,0 +1,40 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{snapshot_nonce::SnapshotNonce, Guard}; + +type InnerIter = Box + Send + 'static>; + +/// A wrapper around iterators that keep a snapshot alive +// +// We need to hold the snapshot nonce so the GC watermark does not +// move past this snapshot nonce, removing data that may still be read. +// +// Additionally, this struct also maps lsm-tree's Guards to "our" Guards. +pub struct Iter { + iter: InnerIter, + + #[expect(unused)] + nonce: SnapshotNonce, +} + +impl Iter { + pub(crate) fn new(nonce: SnapshotNonce, iter: InnerIter) -> Self { + Self { iter, nonce } + } +} + +impl Iterator for Iter { + type Item = crate::Guard; + + fn next(&mut self) -> Option { + self.iter.next().map(Guard) + } +} + +impl DoubleEndedIterator for Iter { + fn next_back(&mut self) -> Option { + self.iter.next_back().map(Guard) + } +} diff --git a/crates/fjall/src/journal/batch_reader.rs b/crates/fjall/src/journal/batch_reader.rs new file mode 100644 index 000000000..4b858d8f4 --- /dev/null +++ b/crates/fjall/src/journal/batch_reader.rs @@ -0,0 +1,216 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::reader::JournalReader; +use crate::{journal::entry::Entry, keyspace::InternalKeyspaceId, JournalRecoveryError}; +use lsm_tree::{SeqNo, UserKey, UserValue, ValueType}; +use std::{fs::OpenOptions, hash::Hasher}; + +#[derive(Debug)] +pub struct ReadBatchItem { + pub keyspace_id: InternalKeyspaceId, + pub key: UserKey, + pub value: UserValue, + pub value_type: ValueType, +} + +#[derive(Debug)] +pub struct Batch { + pub(crate) seqno: SeqNo, + pub(crate) items: Vec, + pub(crate) cleared_keyspaces: Vec, +} + +#[expect(clippy::module_name_repetitions)] +pub struct JournalBatchReader { + reader: JournalReader, + items: Vec, + cleared_keyspaces: Vec, + is_in_batch: bool, + batch_counter: u32, + batch_seqno: SeqNo, + last_valid_pos: u64, + checksum_builder: xxhash_rust::xxh3::Xxh3, +} + +impl JournalBatchReader { + pub fn new(reader: JournalReader) -> Self { + Self { + reader, + items: Vec::with_capacity(10), + cleared_keyspaces: Vec::new(), + checksum_builder: xxhash_rust::xxh3::Xxh3::new(), + is_in_batch: false, + batch_seqno: 0, + last_valid_pos: 0, + batch_counter: 0, + } + } + + // TODO: reallocate space + fn truncate_to(&self, last_valid_pos: u64) -> crate::Result<()> { + log::trace!("Truncating journal to {last_valid_pos}"); + + // TODO: on windows, reading file probably needs to be closed first...? + + let file = OpenOptions::new().write(true).open(&self.reader.path)?; + file.set_len(last_valid_pos)?; + file.sync_all()?; + + Ok(()) + } + + fn on_close(&self) -> crate::Result<()> { + if self.is_in_batch { + log::debug!("Invalid batch: missing terminator, but last batch, so probably incomplete, discarding to keep atomicity"); + + // Discard batch + self.truncate_to(self.last_valid_pos)?; + } + + Ok(()) + } +} + +impl Iterator for JournalBatchReader { + type Item = crate::Result; + + #[expect(clippy::too_many_lines)] + fn next(&mut self) -> Option { + use crate::Error::JournalRecovery; + + loop { + let Some(item) = self.reader.next() else { + fail_iter!(self.on_close()); + return None; + }; + let item = fail_iter!(item); + + let journal_file_pos = self.reader.last_valid_pos; + + match item { + Entry::Start { item_count, seqno } => { + if self.is_in_batch { + log::debug!("Invalid batch: found batch start inside batch"); + + // Discard batch + fail_iter!(self.truncate_to(self.last_valid_pos)); + + return None; + } + + self.is_in_batch = true; + self.batch_counter = item_count; + self.batch_seqno = seqno; + } + Entry::End(expected_checksum) => { + if self.batch_counter > 0 { + log::error!("Invalid batch: insufficient length"); + return Some(Err(JournalRecovery( + JournalRecoveryError::InsufficientLength, + ))); + } + + if !self.is_in_batch { + log::error!("Invalid batch: found end marker without start marker"); + + // Discard batch + fail_iter!(self.truncate_to(self.last_valid_pos)); + + return None; + } + + let got_checksum = self.checksum_builder.finish(); + self.checksum_builder = xxhash_rust::xxh3::Xxh3::new(); + + if got_checksum != expected_checksum { + log::error!("Invalid batch: checksum check failed, expected: {expected_checksum}, got: {got_checksum}"); + return Some(Err(JournalRecovery(JournalRecoveryError::ChecksumMismatch))); + } + + // Reset all variables + self.is_in_batch = false; + self.batch_counter = 0; + + self.last_valid_pos = journal_file_pos; + + let items = std::mem::take(&mut self.items); + let cleared_keyspaces = std::mem::take(&mut self.cleared_keyspaces); + return Some(Ok(Batch { + seqno: self.batch_seqno, + items, + cleared_keyspaces, + })); + } + Entry::Item { + keyspace_id, + key, + value, + value_type, + compression, + } => { + let item = Entry::Item { + keyspace_id, + key: key.clone(), + value: value.clone(), + value_type, + compression, + }; + let mut bytes = Vec::with_capacity(100); + fail_iter!(item.encode_into(&mut bytes)); + + self.checksum_builder.update(&bytes); + + if !self.is_in_batch { + log::debug!("Invalid batch: found end marker without start marker"); + + // Discard batch + fail_iter!(self.truncate_to(self.last_valid_pos)); + + return None; + } + + if self.batch_counter == 0 { + log::error!("Invalid batch: Expected end marker (too many items in batch)"); + return Some(Err(JournalRecovery(JournalRecoveryError::TooManyItems))); + } + + self.batch_counter -= 1; + + self.items.push(ReadBatchItem { + keyspace_id, + key, + value, + value_type, + }); + } + Entry::Clear { keyspace_id } => { + let entry = Entry::Clear { keyspace_id }; + let mut bytes = Vec::with_capacity(16); + fail_iter!(entry.encode_into(&mut bytes)); + + self.checksum_builder.update(&bytes); + + if !self.is_in_batch { + log::debug!("Invalid batch: found clear marker without start marker"); + + // Discard batch + fail_iter!(self.truncate_to(self.last_valid_pos)); + + return None; + } + + if self.batch_counter == 0 { + log::error!("Invalid batch: Expected end marker (too many items in batch)"); + return Some(Err(JournalRecovery(JournalRecoveryError::TooManyItems))); + } + + self.batch_counter -= 1; + + self.cleared_keyspaces.push(keyspace_id); + } + } + } + } +} diff --git a/crates/fjall/src/journal/entry.rs b/crates/fjall/src/journal/entry.rs new file mode 100644 index 000000000..4b7066c5b --- /dev/null +++ b/crates/fjall/src/journal/entry.rs @@ -0,0 +1,306 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{file::MAGIC_BYTES, keyspace::InternalKeyspaceId, Slice}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use lsm_tree::{ + coding::{Decode, Encode}, + CompressionType, SeqNo, UserKey, UserValue, ValueType, +}; +use std::io::{Read, Write}; + +/// Journal entry. Every batch is composed as a Start, followed by N items, followed by an End. +/// +/// - The start entry contains the numbers of items. If the numbers of items following doesn't match, the batch is broken. +/// +/// - The end entry contains a checksum value. If the checksum of the items doesn't match that, the batch is broken. +/// +/// - The end entry terminates each batch with the magic string: [`TRAILER_MAGIC`]. +/// +/// - If a start entry is detected, while inside a batch, the batch is broken. +#[derive(Debug, Eq, PartialEq)] +pub enum Entry { + Start { + item_count: u32, + seqno: SeqNo, + }, + Item { + keyspace_id: InternalKeyspaceId, + key: UserKey, + value: UserValue, + value_type: ValueType, + compression: CompressionType, + }, + End(u64), + Clear { + keyspace_id: InternalKeyspaceId, + }, +} + +pub fn serialize_marker_item( + writer: &mut W, + keyspace_id: InternalKeyspaceId, + key: &[u8], + value: &[u8], + value_type: ValueType, + compression: CompressionType, +) -> Result<(), lsm_tree::Error> { + writer.write_u8(Tag::Item.into())?; + + writer.write_u8(u8::from(value_type))?; + + compression.encode_into(writer)?; + + let compressed_value = match compression { + CompressionType::None => std::borrow::Cow::Borrowed(value), + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => { + let compressed = lz4_flex::compress(value); + std::borrow::Cow::Owned(compressed) + } + }; + + // NOTE: Truncation is okay and actually needed + writer.write_u64::(keyspace_id)?; + + // NOTE: Truncation is okay and actually needed + #[expect(clippy::cast_possible_truncation)] + writer.write_u16::(key.len() as u16)?; + + // NOTE: Truncation is okay and actually needed + #[expect(clippy::cast_possible_truncation)] + writer.write_u32::(value.len() as u32)?; + + // NOTE: Truncation is okay and actually needed + #[expect(clippy::cast_possible_truncation)] + writer.write_u32::(compressed_value.len() as u32)?; + + writer.write_all(key)?; + + writer.write_all(&compressed_value)?; + + Ok(()) +} + +pub enum Tag { + Start = 1, + Item = 2, + End = 3, + Clear = 4, +} + +impl TryFrom for Tag { + type Error = crate::Error; + + fn try_from(value: u8) -> Result { + use Tag::{Clear, End, Item, Start}; + + match value { + 1 => Ok(Start), + 2 => Ok(Item), + 3 => Ok(End), + 4 => Ok(Clear), + _ => Err(crate::Error::InvalidTag(("JournalMarkerTag", value))), + } + } +} + +impl From for u8 { + fn from(val: Tag) -> Self { + val as Self + } +} + +impl Entry { + #[cfg(test)] + pub fn encode_into_vec(&self) -> Vec { + let mut buf = Vec::new(); + self.encode_into(&mut buf).expect("should encode"); + buf + } + + pub(crate) fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error> { + use Entry::{Clear, End, Item, Start}; + + match self { + Start { item_count, seqno } => { + writer.write_u8(Tag::Start.into())?; + writer.write_u32::(*item_count)?; + writer.write_u64::(*seqno)?; + } + Item { + keyspace_id, + key, + value, + value_type, + compression, + } => { + serialize_marker_item(writer, *keyspace_id, key, value, *value_type, *compression)?; + } + End(val) => { + writer.write_u8(Tag::End.into())?; + writer.write_u64::(*val)?; + + // NOTE: Write some fixed trailer bytes so we know the end marker is fully written + // Otherwise we couldn't know if the checksum value is maybe mangled + // (only partially written, with the rest being padding zeroes) + writer.write_all(MAGIC_BYTES)?; + } + Clear { keyspace_id } => { + writer.write_u8(Tag::Clear.into())?; + writer.write_u64::(*keyspace_id)?; + } + } + Ok(()) + } + + pub(crate) fn decode_from(reader: &mut R) -> Result { + match reader.read_u8()?.try_into()? { + Tag::Start => { + let item_count = reader.read_u32::()?; + let seqno = reader.read_u64::()?; + Ok(Self::Start { item_count, seqno }) + } + Tag::Item => { + let value_type = reader.read_u8()?; + let value_type = value_type + .try_into() + .map_err(|()| lsm_tree::Error::InvalidTag(("ValueType", value_type)))?; + + let compression = CompressionType::decode_from(reader)?; + + // Read keyspace ID + let keyspace_id = reader.read_u64::()?; + + // Read key len + let key_len = reader.read_u16::()?; + + // Read real value size + let value_len = reader.read_u32::()?; + + // Read on-disk value size + let on_disk_value_len = reader.read_u32::()?; + + let key = Slice::from_reader(reader, usize::from(key_len))?; + + let value = match compression { + CompressionType::None => { + debug_assert_eq!(value_len, on_disk_value_len); + Slice::from_reader(reader, on_disk_value_len as usize)? + } + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => { + let compressed_value = + Slice::from_reader(reader, on_disk_value_len as usize)?; + + #[warn(unsafe_code)] + let mut value = unsafe { Slice::builder_unzeroed(value_len as usize) }; + + let size = lz4_flex::decompress_into(&compressed_value, &mut value) + .map_err(|e| { + log::error!("LZ4 decompression failed: {e}"); + crate::Error::Decompress(CompressionType::Lz4) + })?; + + if size != value.len() { + log::error!("Decompressed size does not match expected value size"); + return Err(crate::Error::Decompress(CompressionType::Lz4)); + } + + Slice::from(value.freeze()) + } + }; + + Ok(Self::Item { + keyspace_id, + key, + value, + value_type, + compression, + }) + } + Tag::End => { + let checksum = reader.read_u64::()?; + + // Check trailer + let mut magic = [0u8; MAGIC_BYTES.len()]; + reader.read_exact(&mut magic)?; + + if magic != MAGIC_BYTES { + return Err(crate::Error::InvalidTrailer); + } + + Ok(Self::End(checksum)) + } + Tag::Clear => { + let keyspace_id = reader.read_u64::()?; + Ok(Self::Clear { keyspace_id }) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn test_serialize_and_deserialize_success() -> crate::Result<()> { + let item = Entry::Item { + keyspace_id: 0, + key: vec![1, 2, 3].into(), + value: vec![].into(), + value_type: ValueType::Value, + compression: CompressionType::None, + }; + + let serialized_data = item.encode_into_vec(); + let mut reader = &serialized_data[..]; + let deserialized_item = Entry::decode_from(&mut reader)?; + + assert_eq!(item, deserialized_item); + + Ok(()) + } + + #[test] + fn test_invalid_deserialize() { + let invalid_data = [Tag::Start as u8; 1]; // Should be followed by a u32 + + // Try to deserialize with invalid data + let mut reader = &invalid_data[..]; + let result = Entry::decode_from(&mut reader); + + match result { + Ok(_) => panic!("should error"), + Err(error) => match error { + crate::Error::Io(e) => match e.kind() { + std::io::ErrorKind::UnexpectedEof => {} + _ => panic!("should throw UnexpectedEof"), + }, + _ => panic!("should throw UnexpectedEof"), + }, + } + } + + #[test] + fn test_invalid_tag() { + let invalid_data = [5u8; 1]; // Invalid tag + + // Try to deserialize with invalid data + let mut reader = &invalid_data[..]; + let result = Entry::decode_from(&mut reader); + + match result { + Ok(_) => panic!("should error"), + Err(error) => match error { + crate::Error::InvalidTag(("JournalMarkerTag", 5)) => {} + _ => panic!("should throw InvalidTag"), + }, + } + } +} diff --git a/crates/fjall/src/journal/error.rs b/crates/fjall/src/journal/error.rs new file mode 100644 index 000000000..78fc0cc79 --- /dev/null +++ b/crates/fjall/src/journal/error.rs @@ -0,0 +1,30 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Errors that can occur during journal recovery +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[expect(clippy::module_name_repetitions)] +pub enum RecoveryError { + /// Batch had less items than expected, so it's incomplete + InsufficientLength, + + /* /// Batch was not terminated, so it's possibly incomplete + MissingTerminator, */ + /// Too many items in batch + TooManyItems, + + /// The checksum value does not match the expected value + ChecksumMismatch, + + /// An unparseable journal file name was encountered + InvalidFileName, +} + +impl std::fmt::Display for RecoveryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "RecoveryError({self:?})") + } +} + +impl std::error::Error for RecoveryError {} diff --git a/crates/fjall/src/journal/manager.rs b/crates/fjall/src/journal/manager.rs new file mode 100644 index 000000000..3fad1b533 --- /dev/null +++ b/crates/fjall/src/journal/manager.rs @@ -0,0 +1,186 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::writer::Writer; +use crate::Keyspace; +use lsm_tree::{AbstractTree, SeqNo}; +use std::{path::PathBuf, sync::MutexGuard}; + +/// Stores the highest seqno of a keyspace found in a journal. +#[derive(Clone)] +pub struct EvictionWatermark { + pub(crate) keyspace: Keyspace, + pub(crate) lsn: SeqNo, +} + +impl std::fmt::Debug for EvictionWatermark { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.keyspace.name, self.lsn) + } +} + +pub struct Item { + pub(crate) path: PathBuf, + pub(crate) size_in_bytes: u64, + pub(crate) watermarks: Vec, +} + +impl std::fmt::Debug for Item { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "JournalManagerItem {:?} => {:#?}", + self.path, self.watermarks + ) + } +} + +/// The [`JournalManager`] keeps track of sealed journals that are being flushed. +/// +/// Each journal may contain items of different keyspaces. +#[expect(clippy::module_name_repetitions)] +#[derive(Debug)] +pub struct JournalManager { + items: Vec, + disk_space_in_bytes: u64, +} + +impl Drop for JournalManager { + fn drop(&mut self) { + log::trace!("Dropping journal manager"); + + #[cfg(feature = "__internal_whitebox")] + crate::drop::decrement_drop_counter(); + } +} + +impl JournalManager { + pub(crate) fn new() -> Self { + #[cfg(feature = "__internal_whitebox")] + crate::drop::increment_drop_counter(); + + Self { + items: Vec::with_capacity(10), + disk_space_in_bytes: 0, + } + } + + pub(crate) fn clear(&mut self) { + self.items.clear(); + } + + pub(crate) fn enqueue(&mut self, item: Item) { + self.disk_space_in_bytes = self.disk_space_in_bytes.saturating_add(item.size_in_bytes); + self.items.push(item); + } + + /// Returns the number of journals + pub(crate) fn journal_count(&self) -> usize { + // NOTE: + 1 = active journal + self.sealed_journal_count() + 1 + } + + /// Returns the number of sealed journals + pub(crate) fn sealed_journal_count(&self) -> usize { + self.items.len() + } + + /// Returns the number of bytes used on disk by journals + pub(crate) fn disk_space_used(&self) -> u64 { + self.disk_space_in_bytes + } + + /// Gets keyspaces to be flushed so that the oldest journal can be safely evicted + pub(crate) fn get_keyspaces_to_flush_for_oldest_journal_eviction(&self) -> Vec { + let mut items = vec![]; + + if let Some(item) = self.items.first() { + for item in &item.watermarks { + let Some(partition_seqno) = item.keyspace.tree.get_highest_persisted_seqno() else { + items.push(item.keyspace.clone()); + continue; + }; + + if partition_seqno < item.lsn { + items.push(item.keyspace.clone()); + } + } + } + + items + } + + /// Performs maintenance, maybe deleting some old journals + pub(crate) fn maintenance(&mut self) -> crate::Result<()> { + log::debug!("Running journal maintenance"); + + loop { + let Some(item) = self.items.first() else { + return Ok(()); + }; + + // TODO: unit test: check deleted keyspace does not prevent journal eviction + for item in &item.watermarks { + // Only check keyspace seqno if not deleted + if !item + .keyspace + .is_deleted + .load(std::sync::atomic::Ordering::Acquire) + { + let Some(keyspace_seqno) = item.keyspace.tree.get_highest_persisted_seqno() + else { + return Ok(()); + }; + + if keyspace_seqno < item.lsn { + log::trace!( + "Keyspace {:?} not flushed enough to evict journal", + item.keyspace.name, + ); + return Ok(()); + } + } + } + + // NOTE: Once the LSN of *every* keyspace's tables [1] is higher than the journal's stored keyspace seqno, + // it can be deleted from disk, as we know the entire journal has been flushed to tables [2]. + // + // [1] We cannot use the keyspace's max seqno, because the memtable will get writes, which increase the seqno. + // We *need* to check the tables specifically, they are the source of truth for flushed data. + // + // [2] Checking the seqno is safe because the queues inside the flush manager are FIFO. + // + // IMPORTANT: On recovery, the journals need to be flushed from oldest to newest. + log::trace!("Removing fully flushed journal at {}", item.path.display()); + + std::fs::remove_file(&item.path).inspect_err(|e| { + log::error!( + "Failed to clean up stale journal file at {}: {e:?}", + item.path.display(), + ); + })?; + + self.disk_space_in_bytes = self.disk_space_in_bytes.saturating_sub(item.size_in_bytes); + self.items.remove(0); + } + } + + pub(crate) fn rotate_journal( + &mut self, + journal_writer: &mut MutexGuard, + watermarks: Vec, + ) -> crate::Result<()> { + let journal_size = journal_writer.len()?; + + let (sealed_path, _) = journal_writer.rotate()?; + + self.enqueue(Item { + path: sealed_path, + watermarks, + size_in_bytes: journal_size, + }); + + Ok(()) + } +} diff --git a/crates/fjall/src/journal/mod.rs b/crates/fjall/src/journal/mod.rs new file mode 100644 index 000000000..7b656d079 --- /dev/null +++ b/crates/fjall/src/journal/mod.rs @@ -0,0 +1,134 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod batch_reader; +pub mod entry; +pub mod error; +pub mod manager; +pub mod reader; +mod recovery; +pub mod writer; + +#[cfg(test)] +mod test; + +use self::writer::PersistMode; +use crate::file::fsync_directory; +use batch_reader::JournalBatchReader; +use lsm_tree::CompressionType; +use reader::JournalReader; +use recovery::{recover_journals, RecoveryResult}; +use std::{ + path::{Path, PathBuf}, + sync::{Mutex, MutexGuard}, +}; +use writer::Writer; + +pub struct Journal { + writer: Mutex, +} + +impl std::fmt::Debug for Journal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + self.path() + .map(|p| p.display().to_string()) + .unwrap_or_else(|_| String::from("")) + ) + } +} + +impl Drop for Journal { + fn drop(&mut self) { + log::trace!("Dropping journal, trying to flush"); + + match self.persist(PersistMode::SyncAll) { + Ok(()) => { + log::trace!("Flushed journal successfully"); + } + Err(e) => { + log::error!("Flush error on drop: {e:?}"); + } + } + + #[cfg(feature = "__internal_whitebox")] + crate::drop::decrement_drop_counter(); + } +} + +impl Journal { + pub fn with_compression(self, comp: CompressionType, threshold: usize) -> Self { + { + let mut writer = self.writer.lock().expect("lock is poisoned"); + writer.set_compression(comp, threshold); + } + self + } + + fn from_file>(path: P) -> crate::Result { + Ok(Self { + writer: Mutex::new(Writer::from_file(path)?), + }) + } + + pub fn create_new>(path: P) -> crate::Result { + let path = path.as_ref(); + log::trace!("Creating new journal at {}", path.display()); + + let folder = path.parent().expect("parent should exist"); + + std::fs::create_dir_all(folder).inspect_err(|e| { + log::error!( + "Failed to create journal folder at {}: {e:?}", + path.display(), + ); + })?; + + let writer = Writer::create_new(path)?; + + // IMPORTANT: fsync folder on Unix + fsync_directory(folder)?; + + #[cfg(feature = "__internal_whitebox")] + crate::drop::increment_drop_counter(); + + Ok(Self { + writer: Mutex::new(writer), + }) + } + + /// Hands out write access to the journal. + /// + /// # Errors + /// + /// Returns an error if the journal writer is poisoned. + pub(crate) fn get_writer(&self) -> crate::Result> { + self.writer.lock().map_err(|_| crate::Error::Poisoned) + } + + pub fn path(&self) -> crate::Result { + Ok(self.get_writer()?.path.clone()) + } + + pub fn get_reader(&self) -> crate::Result { + let raw_reader = JournalReader::new(self.path()?)?; + Ok(JournalBatchReader::new(raw_reader)) + } + + /// Persists the journal. + pub fn persist(&self, mode: PersistMode) -> crate::Result<()> { + let mut journal_writer = self.get_writer()?; + journal_writer.persist(mode).map_err(Into::into) + } + + pub fn recover>( + path: P, + compression: CompressionType, + compression_threshold: usize, + ) -> crate::Result { + recover_journals(path, compression, compression_threshold) + } +} diff --git a/crates/fjall/src/journal/reader.rs b/crates/fjall/src/journal/reader.rs new file mode 100644 index 000000000..e1cb6eb3b --- /dev/null +++ b/crates/fjall/src/journal/reader.rs @@ -0,0 +1,78 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::entry::Entry; +use std::{ + fs::{File, OpenOptions}, + io::{BufReader, Seek}, + path::{Path, PathBuf}, +}; + +/// Reads and emits through the entries in a journal file, but doesn't +/// check the validity of batches +/// +/// Will truncate the file to the last valid position to prevent corrupt +/// bytes at the end of the file, which would jeopardize future writes into the file. +#[expect(clippy::module_name_repetitions)] +pub struct JournalReader { + pub(crate) path: PathBuf, + pub(crate) reader: BufReader, + pub(crate) last_valid_pos: u64, +} + +impl JournalReader { + pub fn new>(path: P) -> crate::Result { + let file = OpenOptions::new().read(true).write(true).open(&path)?; + + Ok(Self { + path: path.as_ref().into(), + reader: BufReader::new(file), + last_valid_pos: 0, + }) + } + + fn truncate_file(&mut self, pos: u64) -> crate::Result<()> { + log::debug!("truncating journal to {pos}"); + self.reader.get_mut().set_len(pos)?; + self.reader.get_mut().sync_all()?; + Ok(()) + } + + fn maybe_truncate_file_to_last_valid_pos(&mut self) -> crate::Result<()> { + let stream_pos = self.reader.stream_position()?; + + if stream_pos > self.last_valid_pos { + self.truncate_file(self.last_valid_pos)?; + } + + Ok(()) + } +} + +impl Iterator for JournalReader { + type Item = crate::Result; + + fn next(&mut self) -> Option { + match Entry::decode_from(&mut self.reader) { + Ok(item) => { + self.last_valid_pos = fail_iter!(self.reader.stream_position()); + Some(Ok(item)) + } + Err(e) => { + if let crate::Error::Io(e) = e { + match e.kind() { + std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::Other => { + fail_iter!(self.maybe_truncate_file_to_last_valid_pos()); + None + } + _ => Some(Err(crate::Error::Io(e))), + } + } else { + fail_iter!(self.maybe_truncate_file_to_last_valid_pos()); + None + } + } + } + } +} diff --git a/crates/fjall/src/journal/recovery.rs b/crates/fjall/src/journal/recovery.rs new file mode 100644 index 000000000..d3a12914d --- /dev/null +++ b/crates/fjall/src/journal/recovery.rs @@ -0,0 +1,91 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::Journal; +use lsm_tree::CompressionType; +use std::path::{Path, PathBuf}; + +pub type JournalId = u64; + +#[derive(Debug)] +pub struct RecoveryResult { + pub(crate) active: Journal, + pub(crate) sealed: Vec<(JournalId, PathBuf)>, + pub(crate) was_active_created: bool, +} + +pub fn recover_journals>( + path: P, + compression: CompressionType, + compression_threshold: usize, +) -> crate::Result { + let path = path.as_ref(); + + let mut max_journal_id: JournalId = 0; + let mut journal_fragments = Vec::<(JournalId, PathBuf)>::new(); + + log::trace!("Got journal fragments: {journal_fragments:#?}"); + + for dirent in std::fs::read_dir(path)? { + let dirent = dirent?; + let path = dirent.path(); + let filename = dirent.file_name(); + + let Some(filename) = filename.to_str() else { + log::error!("Invalid journal file name: {}", filename.display()); + return Err(crate::Error::JournalRecovery( + crate::JournalRecoveryError::InvalidFileName, + )); + }; + + if !std::path::Path::new(filename) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("jnl")) + { + continue; + } + + assert!(dirent.file_type()?.is_file()); + + let Some(basename) = filename.strip_suffix(".jnl") else { + log::error!("Invalid journal file name: {filename}"); + return Err(crate::Error::JournalRecovery( + crate::JournalRecoveryError::InvalidFileName, + )); + }; + + let journal_id = basename.parse::().map_err(|_| { + log::error!("Invalid journal file name: {filename}"); + crate::Error::JournalRecovery(crate::JournalRecoveryError::InvalidFileName) + })?; + + max_journal_id = max_journal_id.max(journal_id); + + journal_fragments.push((journal_id, path)); + } + + // NOTE: Sort ascending, so the last item is the active journal + journal_fragments.sort_by_key(|(a, _)| *a); + + log::trace!("Recovered {journal_fragments:#?}"); + + Ok(match journal_fragments.pop() { + Some((_, active)) => RecoveryResult { + active: Journal::from_file(active)? + .with_compression(compression, compression_threshold), + sealed: journal_fragments, + was_active_created: false, + }, + None => RecoveryResult { + active: { + let id: JournalId = max_journal_id + 1; + + Journal::create_new(path.join(id.to_string()))? + .with_compression(compression, compression_threshold) + }, + sealed: vec![], + was_active_created: true, + }, + }) +} diff --git a/crates/fjall/src/journal/test.rs b/crates/fjall/src/journal/test.rs new file mode 100644 index 000000000..3d51e4dcb --- /dev/null +++ b/crates/fjall/src/journal/test.rs @@ -0,0 +1,488 @@ +use super::*; +use crate::batch::item::Item as BatchItem; +use entry::Entry; +use lsm_tree::ValueType; +use std::io::Write; +use tempfile::tempdir; +use test_log::test; + +impl PartialEq for crate::journal::batch_reader::ReadBatchItem { + fn eq(&self, other: &BatchItem) -> bool { + self.keyspace_id == other.keyspace.id + && self.key == other.key + && self.value == other.value + && self.value_type == other.value_type + } +} + +impl PartialEq for BatchItem { + fn eq(&self, other: &crate::journal::batch_reader::ReadBatchItem) -> bool { + other.eq(self) + } +} + +#[test] +#[expect(clippy::redundant_clone)] +fn journal_rotation() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace = db.keyspace("default", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + let next_path = dir2.path().join("1.jnl"); + + { + let journal = Journal::create_new(&path)?; + let mut writer = journal.get_writer()?; + + writer.write_batch( + [ + BatchItem::new(keyspace.clone(), *b"a", *b"a", ValueType::Value), + BatchItem::new(keyspace.clone(), *b"b", *b"b", ValueType::Value), + ] + .iter(), + 2, + 0, + )?; + writer.rotate()?; + } + + assert!(path.try_exists()?); + assert!(next_path.try_exists()?); + + Ok(()) +} + +#[test] +#[expect(clippy::redundant_clone)] +fn journal_recovery_active() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace0 = db.keyspace("default", Default::default)?; + let keyspace1 = db.keyspace("default1", Default::default)?; + let keyspace2 = db.keyspace("default2", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + let next_path = dir2.path().join("1.jnl"); + let next_next_path = dir2.path().join("2.jnl"); + + { + let journal = Journal::create_new(&path)?; + let mut writer = journal.get_writer()?; + + writer.write_batch( + [ + BatchItem::new(keyspace0.clone(), *b"a", *b"a", ValueType::Value), + BatchItem::new(keyspace0.clone(), *b"b", *b"b", ValueType::Value), + ] + .iter(), + 2, + 0, + )?; + writer.rotate()?; + + writer.write_batch( + [ + BatchItem::new(keyspace1.clone(), *b"c", *b"c", ValueType::Value), + BatchItem::new(keyspace1.clone(), *b"d", *b"d", ValueType::Value), + ] + .iter(), + 2, + 1, + )?; + writer.rotate()?; + + writer.write_batch( + [ + BatchItem::new(keyspace2.clone(), *b"c", *b"c", ValueType::Value), + BatchItem::new(keyspace2.clone(), *b"d", *b"d", ValueType::Value), + ] + .iter(), + 2, + 1, + )?; + } + + assert!(path.try_exists()?); + assert!(next_path.try_exists()?); + assert!(next_next_path.try_exists()?); + + let journal_recovered = Journal::recover(dir2, CompressionType::None, 0)?; + assert_eq!(journal_recovered.active.path()?, next_next_path); + assert_eq!(journal_recovered.sealed, &[(0, path), (1, next_path)]); + + Ok(()) +} + +#[test] +#[cfg(feature = "lz4")] +#[expect(clippy::redundant_clone)] +fn journal_recovery_active_lz4() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace0 = db.keyspace("default", Default::default)?; + let keyspace1 = db.keyspace("default1", Default::default)?; + let keyspace2 = db.keyspace("default2", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + let next_path = dir2.path().join("1.jnl"); + let next_next_path = dir2.path().join("2.jnl"); + + { + let journal = Journal::create_new(&path)?.with_compression(CompressionType::Lz4, 1); + let mut writer = journal.get_writer()?; + + writer.write_batch( + [ + BatchItem::new(keyspace0.clone(), *b"a", *b"a", ValueType::Value), + BatchItem::new(keyspace0.clone(), *b"b", *b"b", ValueType::Value), + ] + .iter(), + 2, + 0, + )?; + writer.rotate()?; + + writer.write_batch( + [ + BatchItem::new(keyspace1.clone(), *b"c", *b"c", ValueType::Value), + BatchItem::new(keyspace1.clone(), *b"d", *b"d", ValueType::Value), + ] + .iter(), + 2, + 1, + )?; + writer.rotate()?; + + writer.write_batch( + [ + BatchItem::new(keyspace2.clone(), *b"c", *b"c", ValueType::Value), + BatchItem::new(keyspace2.clone(), *b"d", *b"d", ValueType::Value), + ] + .iter(), + 2, + 1, + )?; + } + + assert!(path.try_exists()?); + assert!(next_path.try_exists()?); + assert!(next_next_path.try_exists()?); + + let journal_recovered = Journal::recover(dir2, CompressionType::None, 0)?; + assert_eq!(journal_recovered.active.path()?, next_next_path); + assert_eq!(journal_recovered.sealed, &[(0, path), (1, next_path)]); + + Ok(()) +} + +#[test] +#[expect(clippy::redundant_clone)] +fn journal_recovery_no_active() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace = db.keyspace("default", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + let next_path = dir2.path().join("1.jnl"); + + { + let journal = Journal::create_new(&path)?; + + { + let mut writer = journal.get_writer()?; + + writer.write_batch( + [ + BatchItem::new(keyspace.clone(), *b"a", *b"a", ValueType::Value), + BatchItem::new(keyspace.clone(), *b"b", *b"b", ValueType::Value), + ] + .iter(), + 2, + 0, + )?; + writer.rotate()?; + } + + // NOTE: Delete the new, active journal -> old journal will be + // reused as active on next recovery + std::fs::remove_file(&next_path)?; + } + + assert!(path.try_exists()?); + assert!(!next_path.try_exists()?); + + let journal_recovered = Journal::recover(dir2, CompressionType::None, 0)?; + assert_eq!(journal_recovered.active.path()?, path); + assert_eq!(journal_recovered.sealed, &[]); + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used, clippy::redundant_clone)] +fn journal_truncation_corrupt_bytes() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace = db.keyspace("default", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + + let values = [ + BatchItem::new(keyspace.clone(), *b"abc", *b"def", ValueType::Value), + BatchItem::new(keyspace.clone(), *b"yxc", *b"ghj", ValueType::Value), + ]; + + { + let journal = Journal::create_new(&path)?; + journal + .get_writer()? + .write_batch(values.iter(), values.len(), 0)?; + } + + { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + file.write_all(b"09pmu35w3a9mp53bao9upw3ab5up")?; + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + for _ in 0..5 { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + file.write_all(b"09pmu35w3a9mp53bao9upw3ab5up")?; + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used, clippy::redundant_clone)] +fn journal_truncation_repeating_start_marker() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace = db.keyspace("default", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + + let values = [ + BatchItem::new(keyspace.clone(), *b"abc", *b"def", ValueType::Value), + BatchItem::new(keyspace.clone(), *b"yxc", *b"ghj", ValueType::Value), + ]; + + { + let journal = Journal::create_new(&path)?; + journal + .get_writer()? + .write_batch(values.iter(), values.len(), 0)?; + } + + { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + Entry::Start { + item_count: 2, + seqno: 64, + } + .encode_into(&mut file)?; + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + for _ in 0..5 { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + Entry::Start { + item_count: 2, + seqno: 64, + } + .encode_into(&mut file)?; + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used, clippy::redundant_clone)] +fn journal_truncation_repeating_end_marker() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace = db.keyspace("default", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + + let values = [ + BatchItem::new(keyspace.clone(), *b"abc", *b"def", ValueType::Value), + BatchItem::new(keyspace.clone(), *b"yxc", *b"ghj", ValueType::Value), + ]; + + { + let journal = Journal::create_new(&path)?; + journal + .get_writer()? + .write_batch(values.iter(), values.len(), 0)?; + } + + { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + Entry::End(5432).encode_into(&mut file)?; + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + for _ in 0..5 { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + Entry::End(5432).encode_into(&mut file)?; + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used, clippy::redundant_clone)] +fn journal_truncation_repeating_item_marker() -> crate::Result<()> { + let dir1 = tempdir()?; + let db = crate::Database::builder(&dir1).open()?; + let keyspace = db.keyspace("default", Default::default)?; + + let dir2 = tempdir()?; + let path = dir2.path().join("0.jnl"); + + let values = [ + BatchItem::new(keyspace.clone(), *b"abc", *b"def", ValueType::Value), + BatchItem::new(keyspace.clone(), *b"yxc", *b"ghj", ValueType::Value), + ]; + + { + let journal = Journal::create_new(&path)?; + journal + .get_writer()? + .write_batch(values.iter(), values.len(), 0)?; + } + + { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + Entry::Item { + keyspace_id: 0, + key: (*b"zzz").into(), + value: (*b"").into(), + value_type: ValueType::Tombstone, + compression: lsm_tree::CompressionType::None, + } + .encode_into(&mut file)?; + + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + // Mangle journal + for _ in 0..5 { + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + Entry::Item { + keyspace_id: 0, + key: (*b"zzz").into(), + value: (*b"").into(), + value_type: ValueType::Tombstone, + compression: lsm_tree::CompressionType::None, + } + .encode_into(&mut file)?; + + file.sync_all()?; + } + + for _ in 0..10 { + let journal = Journal::from_file(&path)?; + let reader = journal.get_reader()?; + let collected = reader.flatten().collect::>(); + assert_eq!(values.to_vec(), collected.first().unwrap().items); + } + + Ok(()) +} diff --git a/crates/fjall/src/journal/writer.rs b/crates/fjall/src/journal/writer.rs new file mode 100644 index 000000000..9cac11c52 --- /dev/null +++ b/crates/fjall/src/journal/writer.rs @@ -0,0 +1,380 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::entry::{serialize_marker_item, Entry}; +use crate::{ + batch::item::Item as BatchItem, file::fsync_directory, journal::recovery::JournalId, + keyspace::InternalKeyspaceId, +}; +use lsm_tree::{CompressionType, SeqNo, ValueType}; +use std::{ + fs::{File, OpenOptions}, + hash::Hasher, + io::{BufWriter, Seek, Write}, + path::{Path, PathBuf}, +}; + +// TODO: this should be a database configuration +pub const PRE_ALLOCATED_BYTES: u64 = 64 * 1_024 * 1_024; + +pub const JOURNAL_BUFFER_BYTES: usize = 8 * 1_024; + +pub struct Writer { + pub(crate) path: PathBuf, + file: BufWriter, + buf: Vec, + is_buffer_dirty: bool, + + compression: CompressionType, + compression_threshold: usize, +} + +/// The persist mode allows setting the durability guarantee of previous writes +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum PersistMode { + /// Flushes data to OS buffers. This allows the OS to write out data in case of an + /// application crash. + /// + /// When this function returns, data is **not** guaranteed to be persisted in case + /// of a power loss event or OS crash. + Buffer, + + /// Flushes data using `fdatasync`. + /// + /// Use if you know that `fdatasync` is sufficient for your file system and/or operating system. + SyncData, + + /// Flushes data + metadata using `fsync`. + SyncAll, +} + +impl Writer { + pub fn set_compression(&mut self, comp: CompressionType, threshold: usize) { + self.compression = comp; + self.compression_threshold = threshold; + } + + pub fn pos(&mut self) -> crate::Result { + self.file.stream_position().map_err(Into::into) + } + + pub fn len(&self) -> crate::Result { + Ok(self.file.get_ref().metadata()?.len()) + } + + pub fn rotate(&mut self) -> crate::Result<(PathBuf, PathBuf)> { + self.persist(PersistMode::SyncAll)?; + + log::debug!( + "Sealing active journal at {}, len={}B", + self.path.display(), + self.path + .metadata() + .inspect_err(|e| { + log::error!( + "Failed to get file metadata of journal file at {}: {e:?}", + self.path.display() + ); + })? + .len(), + ); + + let prev_path = self.path.clone(); + + let folder = self + .path + .parent() + .expect("should have parent") + .to_path_buf(); + + let Some(basename) = self + .path + .file_name() + .expect("should be valid file name") + .to_str() + .expect("should be valid utf-8") + .strip_suffix(".jnl") + else { + log::error!("Invalid journal file name: {}", self.path.display()); + return Err(crate::Error::JournalRecovery( + crate::JournalRecoveryError::InvalidFileName, + )); + }; + + let journal_id = basename.parse::().map_err(|_| { + log::error!("Invalid journal file name: {}", self.path.display()); + crate::Error::JournalRecovery(crate::JournalRecoveryError::InvalidFileName) + })?; + + let new_path = folder.join(format!("{}.jnl", journal_id + 1)); + log::debug!("Rotating active journal to {}", new_path.display()); + + let comp = self.compression; + let compt = self.compression_threshold; + *self = Self::create_new(new_path.clone())?; + self.set_compression(comp, compt); + + // IMPORTANT: fsync folder on Unix + fsync_directory(&folder)?; + + Ok((prev_path, new_path)) + } + + pub fn create_new>(path: P) -> crate::Result { + let path = path.into(); + + let file = File::create_new(&path).inspect_err(|e| { + log::error!("Failed to create journal file at {}: {e:?}", path.display()); + })?; + + file.set_len(PRE_ALLOCATED_BYTES).inspect_err(|e| { + log::error!( + "Failed to set journal file size to {PRE_ALLOCATED_BYTES}B at {}: {e:?}", + path.display(), + ); + })?; + + file.sync_all().inspect_err(|e| { + log::error!("Failed to fsync journal file at {}: {e:?}", path.display()); + })?; + + Ok(Self { + path, + file: BufWriter::new(file), + buf: Vec::new(), + is_buffer_dirty: false, + compression: CompressionType::None, + compression_threshold: 0, + }) + } + + pub fn from_file>(path: P) -> crate::Result { + let path = path.as_ref(); + + if !path.try_exists()? { + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(path) + .inspect_err(|e| { + log::error!("Failed to create journal file at {}: {e:?}", path.display()); + })?; + + file.set_len(PRE_ALLOCATED_BYTES).inspect_err(|e| { + log::error!( + "Failed to set journal file size to {PRE_ALLOCATED_BYTES}B at {}: {e:?}", + path.display(), + ); + })?; + + file.sync_all().inspect_err(|e| { + log::error!("Failed to fsync journal file at {}: {e:?}", path.display()); + })?; + + return Ok(Self { + path: path.into(), + file: BufWriter::with_capacity(JOURNAL_BUFFER_BYTES, file), + buf: Vec::new(), + is_buffer_dirty: false, + compression: CompressionType::None, + compression_threshold: 0, + }); + } + + let file = OpenOptions::new() + .append(true) + .open(path) + .inspect_err(|e| { + log::error!("Failed to open journal file at {}: {e:?}", path.display()); + })?; + + Ok(Self { + path: path.into(), + file: BufWriter::with_capacity(JOURNAL_BUFFER_BYTES, file), + buf: Vec::new(), + is_buffer_dirty: false, + compression: CompressionType::None, + compression_threshold: 0, + }) + } + + /// Persists the journal file. + pub(crate) fn persist(&mut self, mode: PersistMode) -> std::io::Result<()> { + log::trace!( + "Persisting journal at {} with mode={mode:?}", + self.path.display(), + ); + + if self.is_buffer_dirty { + self.file.flush().inspect_err(|e| { + log::error!( + "Failed to flush journal IO buffers at {}: {e:?}", + self.path.display(), + ); + })?; + self.is_buffer_dirty = false; + } + + match mode { + PersistMode::SyncAll => self.file.get_mut().sync_all().inspect_err(|e| { + log::error!( + "Failed to fsync journal file at {}: {e:?}", + self.path.display(), + ); + }), + PersistMode::SyncData => self.file.get_mut().sync_data().inspect_err(|e| { + log::error!( + "Failed to fsyncdata journal file at {}: {e:?}", + self.path.display(), + ); + }), + PersistMode::Buffer => Ok(()), + } + } + + /// Writes a batch start marker to the journal + fn write_start(&mut self, item_count: u32, seqno: SeqNo) -> Result { + debug_assert!(self.buf.is_empty()); + + Entry::Start { item_count, seqno }.encode_into(&mut self.buf)?; + + self.file.write_all(&self.buf)?; + + Ok(self.buf.len()) + } + + /// Writes a batch end marker to the journal + fn write_end(&mut self, checksum: u64) -> Result { + debug_assert!(self.buf.is_empty()); + + Entry::End(checksum).encode_into(&mut self.buf)?; + + self.file.write_all(&self.buf)?; + + Ok(self.buf.len()) + } + + pub(crate) fn write_raw( + &mut self, + keyspace_id: InternalKeyspaceId, + key: &[u8], + value: &[u8], + value_type: ValueType, + seqno: u64, + ) -> crate::Result { + self.is_buffer_dirty = true; + + let mut hasher = xxhash_rust::xxh3::Xxh3::default(); + let mut byte_count = 0; + + self.buf.clear(); + byte_count += self.write_start(1, seqno)?; + self.buf.clear(); + + serialize_marker_item( + &mut self.buf, + keyspace_id, + key, + value, + value_type, + if self.compression_threshold > 0 && value.len() >= self.compression_threshold { + self.compression + } else { + CompressionType::None + }, + )?; + + self.file.write_all(&self.buf)?; + + hasher.update(&self.buf); + byte_count += self.buf.len(); + + self.buf.clear(); + let checksum = hasher.finish(); + byte_count += self.write_end(checksum)?; + + Ok(byte_count) + } + + pub(crate) fn write_clear( + &mut self, + keyspace_id: InternalKeyspaceId, + seqno: SeqNo, + ) -> crate::Result { + self.is_buffer_dirty = true; + + let mut hasher = xxhash_rust::xxh3::Xxh3::default(); + let mut byte_count = 0; + + self.buf.clear(); + byte_count += self.write_start(1, seqno)?; + self.buf.clear(); + + Entry::Clear { keyspace_id }.encode_into(&mut self.buf)?; + self.file.write_all(&self.buf)?; + hasher.update(&self.buf); + byte_count += self.buf.len(); + + self.buf.clear(); + let checksum = hasher.finish(); + byte_count += self.write_end(checksum)?; + + Ok(byte_count) + } + + pub fn write_batch<'a>( + &mut self, + items: impl Iterator, + batch_size: usize, + seqno: SeqNo, + ) -> crate::Result { + if batch_size == 0 { + return Ok(0); + } + + self.is_buffer_dirty = true; + + self.buf.clear(); + + // NOTE: entries.len() is surely never > u32::MAX + #[expect(clippy::cast_possible_truncation)] + let item_count = batch_size as u32; + + let mut hasher = xxhash_rust::xxh3::Xxh3::default(); + let mut byte_count = 0; + + byte_count += self.write_start(item_count, seqno)?; + self.buf.clear(); + + for item in items { + debug_assert!(self.buf.is_empty()); + + serialize_marker_item( + &mut self.buf, + item.keyspace.id, + &item.key, + &item.value, + item.value_type, + if self.compression_threshold > 0 && item.value.len() >= self.compression_threshold + { + self.compression + } else { + CompressionType::None + }, + )?; + + self.file.write_all(&self.buf)?; + + hasher.update(&self.buf); + byte_count += self.buf.len(); + + self.buf.clear(); + } + + let checksum = hasher.finish(); + byte_count += self.write_end(checksum)?; + + Ok(byte_count) + } +} diff --git a/crates/fjall/src/keyspace/config/block_size.rs b/crates/fjall/src/keyspace/config/block_size.rs new file mode 100644 index 000000000..0ed0bdb0a --- /dev/null +++ b/crates/fjall/src/keyspace/config/block_size.rs @@ -0,0 +1,55 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::config::{DecodeConfig, EncodeConfig}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +impl EncodeConfig for crate::config::BlockSizePolicy { + fn encode(&self) -> crate::Slice { + let mut v = vec![]; + + // NOTE: Policies are limited to 255 entries + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::expect_used)] + v.write_u8(self.len() as u8) + .expect("cannot fail writing into a vec"); + + for item in self.iter() { + #[expect(clippy::expect_used)] + v.write_u32::(*item) + .expect("cannot fail writing into a vec"); + } + + v.into() + } +} + +impl DecodeConfig for crate::config::BlockSizePolicy { + fn decode(mut bytes: &[u8]) -> crate::Result { + let len = bytes.read_u8()?; + + let mut v = vec![]; + + for _ in 0..len { + v.push(bytes.read_u32::()?); + } + + Ok(Self::new(v)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn roundtrip_block_size_policy() -> crate::Result<()> { + let policy = crate::config::BlockSizePolicy::new([1024, 2048, 4096]); + let encoded = policy.encode(); + let decoded = crate::config::BlockSizePolicy::decode(&encoded)?; + assert_eq!(policy, decoded); + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/config/compression.rs b/crates/fjall/src/keyspace/config/compression.rs new file mode 100644 index 000000000..c19149cca --- /dev/null +++ b/crates/fjall/src/keyspace/config/compression.rs @@ -0,0 +1,61 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::config::{DecodeConfig, EncodeConfig}; +use byteorder::{ReadBytesExt, WriteBytesExt}; +use lsm_tree::{ + coding::{Decode, Encode}, + CompressionType, +}; + +impl EncodeConfig for crate::config::CompressionPolicy { + fn encode(&self) -> crate::Slice { + let mut v = vec![]; + + // NOTE: Policies are limited to 255 entries + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::expect_used)] + v.write_u8(self.len() as u8) + .expect("cannot fail writing into a vec"); + + for item in self.iter() { + #[expect(clippy::expect_used)] + item.encode_into(&mut v) + .expect("cannot fail writing into a vec"); + } + + v.into() + } +} + +impl DecodeConfig for crate::config::CompressionPolicy { + fn decode(mut bytes: &[u8]) -> crate::Result { + let len = bytes.read_u8()?; + + let mut v = vec![]; + + for _ in 0..len { + v.push(CompressionType::decode_from(&mut bytes)?); + } + + Ok(Self::new(v)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + #[cfg(feature = "lz4")] + fn roundtrip_compression_policy() -> crate::Result<()> { + let policy = + crate::config::CompressionPolicy::new([CompressionType::None, CompressionType::Lz4]); + let encoded = policy.encode(); + let decoded = crate::config::CompressionPolicy::decode(&encoded)?; + assert_eq!(policy, decoded); + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/config/filter.rs b/crates/fjall/src/keyspace/config/filter.rs new file mode 100644 index 000000000..1b0a91bb3 --- /dev/null +++ b/crates/fjall/src/keyspace/config/filter.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::config::{DecodeConfig, EncodeConfig}; +use byteorder::{ReadBytesExt, WriteBytesExt}; + +impl EncodeConfig for crate::config::FilterPolicy { + fn encode(&self) -> crate::Slice { + let mut v = vec![]; + + // NOTE: Policies are limited to 255 entries + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::expect_used)] + v.write_u8(self.len() as u8) + .expect("cannot fail writing into a vec"); + + for item in self.iter() { + match item { + crate::config::FilterPolicyEntry::None => { + v.write_u8(0).expect("cannot fail writing into a vec"); + } + crate::config::FilterPolicyEntry::Bloom(bloom) => { + v.write_u8(1).expect("cannot fail writing into a vec"); + + match bloom { + crate::config::BloomConstructionPolicy::BitsPerKey(bits) => { + v.write_u8(0).expect("cannot fail writing into a vec"); + v.write_f32::(*bits) + .expect("cannot fail writing into a vec"); + } + crate::config::BloomConstructionPolicy::FalsePositiveRate(fpr) => { + v.write_u8(1).expect("cannot fail writing into a vec"); + v.write_f32::(*fpr) + .expect("cannot fail writing into a vec"); + } + } + } + } + } + + v.into() + } +} + +impl DecodeConfig for crate::config::FilterPolicy { + fn decode(mut bytes: &[u8]) -> crate::Result { + let len = bytes.read_u8()?; + + let mut v = vec![]; + + for _ in 0..len { + let tag = bytes.read_u8()?; + + match tag { + 0 => { + v.push(crate::config::FilterPolicyEntry::None); + } + 1 => { + let policy_type = bytes.read_u8()?; + + let policy = match policy_type { + 0 => { + let bits = bytes.read_f32::()?; + + crate::config::FilterPolicyEntry::Bloom( + crate::config::BloomConstructionPolicy::BitsPerKey(bits), + ) + } + 1 => { + let value = bytes.read_f32::()?; + + crate::config::FilterPolicyEntry::Bloom( + crate::config::BloomConstructionPolicy::FalsePositiveRate(value), + ) + } + _ => { + panic!("unknown bloom filter policy type: {policy_type}"); + } + }; + + v.push(policy); + } + _ => { + panic!("unknown filter policy tag: {tag}"); + } + } + } + + Ok(Self::new(v)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn roundtrip_filter_policy() -> crate::Result<()> { + let policy = crate::config::FilterPolicy::new([ + crate::config::FilterPolicyEntry::Bloom( + crate::config::BloomConstructionPolicy::BitsPerKey(10.0), + ), + crate::config::FilterPolicyEntry::Bloom( + crate::config::BloomConstructionPolicy::FalsePositiveRate(0.01), + ), + crate::config::FilterPolicyEntry::None, + ]); + + let encoded = policy.encode(); + let decoded = crate::config::FilterPolicy::decode(&encoded)?; + + assert_eq!(policy, decoded); + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/config/hash_ratio.rs b/crates/fjall/src/keyspace/config/hash_ratio.rs new file mode 100644 index 000000000..939c1a471 --- /dev/null +++ b/crates/fjall/src/keyspace/config/hash_ratio.rs @@ -0,0 +1,55 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::config::{DecodeConfig, EncodeConfig}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +impl EncodeConfig for crate::config::HashRatioPolicy { + fn encode(&self) -> crate::Slice { + let mut v = vec![]; + + // NOTE: Policies are limited to 255 entries + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::expect_used)] + v.write_u8(self.len() as u8) + .expect("cannot fail writing into a vec"); + + for item in self.iter() { + #[expect(clippy::expect_used)] + v.write_f32::(*item) + .expect("cannot fail writing into a vec"); + } + + v.into() + } +} + +impl DecodeConfig for crate::config::HashRatioPolicy { + fn decode(mut bytes: &[u8]) -> crate::Result { + let len = bytes.read_u8()?; + + let mut v = vec![]; + + for _ in 0..len { + v.push(bytes.read_f32::()?); + } + + Ok(Self::new(v)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn roundtrip_hash_ratio_policy() -> crate::Result<()> { + let policy = crate::config::HashRatioPolicy::new([0.1, 0.2, 0.3]); + let encoded = policy.encode(); + let decoded = crate::config::HashRatioPolicy::decode(&encoded)?; + assert_eq!(policy, decoded); + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/config/mod.rs b/crates/fjall/src/keyspace/config/mod.rs new file mode 100644 index 000000000..e7e5b4266 --- /dev/null +++ b/crates/fjall/src/keyspace/config/mod.rs @@ -0,0 +1,20 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod block_size; +mod compression; +mod filter; +mod hash_ratio; +mod pinning; +mod restart_interval; + +pub trait EncodeConfig { + fn encode(&self) -> crate::Slice; +} + +pub trait DecodeConfig { + fn decode(bytes: &[u8]) -> crate::Result + where + Self: Sized; +} diff --git a/crates/fjall/src/keyspace/config/pinning.rs b/crates/fjall/src/keyspace/config/pinning.rs new file mode 100644 index 000000000..9a05be68a --- /dev/null +++ b/crates/fjall/src/keyspace/config/pinning.rs @@ -0,0 +1,54 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::config::{DecodeConfig, EncodeConfig}; +use byteorder::{ReadBytesExt, WriteBytesExt}; + +impl EncodeConfig for crate::config::PinningPolicy { + fn encode(&self) -> crate::Slice { + let mut v = vec![]; + + // NOTE: Policies are limited to 255 entries + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::expect_used)] + v.write_u8(self.len() as u8).expect("cannot fail"); + + for item in self.iter() { + #[expect(clippy::expect_used)] + v.write_u8(u8::from(*item)).expect("cannot fail"); + } + + v.into() + } +} + +impl DecodeConfig for crate::config::PinningPolicy { + fn decode(mut bytes: &[u8]) -> crate::Result { + let len = bytes.read_u8().expect("cannot fail"); + + let mut v = vec![]; + + for _ in 0..len { + let b = bytes.read_u8().expect("cannot fail"); + v.push(b == 1); + } + + Ok(Self::new(v)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn roundtrip_pinning_policy() -> crate::Result<()> { + let policy = crate::config::PinningPolicy::new([true, true, false]); + let encoded = policy.encode(); + let decoded = crate::config::PinningPolicy::decode(&encoded)?; + assert_eq!(policy, decoded); + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/config/restart_interval.rs b/crates/fjall/src/keyspace/config/restart_interval.rs new file mode 100644 index 000000000..61281d277 --- /dev/null +++ b/crates/fjall/src/keyspace/config/restart_interval.rs @@ -0,0 +1,54 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::config::{DecodeConfig, EncodeConfig}; +use byteorder::{ReadBytesExt, WriteBytesExt}; + +impl EncodeConfig for crate::config::RestartIntervalPolicy { + fn encode(&self) -> crate::Slice { + let mut v = vec![]; + + // NOTE: Policies are limited to 255 entries + #[expect(clippy::cast_possible_truncation)] + #[expect(clippy::expect_used)] + v.write_u8(self.len() as u8) + .expect("cannot fail writing into a vec"); + + for item in self.iter() { + #[expect(clippy::expect_used)] + v.write_u8(*item).expect("cannot fail writing into a vec"); + } + + v.into() + } +} + +impl DecodeConfig for crate::config::RestartIntervalPolicy { + fn decode(mut bytes: &[u8]) -> crate::Result { + let len = bytes.read_u8()?; + + let mut v = vec![]; + + for _ in 0..len { + v.push(bytes.read_u8()?); + } + + Ok(Self::new(v)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn roundtrip_restart_interval_policy() -> crate::Result<()> { + let policy = crate::config::RestartIntervalPolicy::new([8, 12, 16]); + let encoded = policy.encode(); + let decoded = crate::config::RestartIntervalPolicy::decode(&encoded)?; + assert_eq!(policy, decoded); + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/mod.rs b/crates/fjall/src/keyspace/mod.rs new file mode 100644 index 000000000..c2573d336 --- /dev/null +++ b/crates/fjall/src/keyspace/mod.rs @@ -0,0 +1,1178 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod config; +pub mod name; +pub mod options; +mod write_delay; + +#[cfg(test)] +mod test; + +use crate::{ + file::{KEYSPACES_FOLDER, LSM_CURRENT_VERSION_MARKER}, + flush::Task as FlushTask, + ingestion::Ingestion, + locked_file::LockedFileGuard, + poison::PoisonSignal, + supervisor::Supervisor, + worker_pool::WorkerMessage, + Database, Guard, Iter, +}; +use lsm_tree::{AbstractTree, AnyTree, SeqNo, UserKey, UserValue}; +use options::CreateOptions; +use std::{ + ops::RangeBounds, + path::Path, + sync::{atomic::AtomicBool, Arc, MutexGuard}, + time::Duration, +}; +use write_delay::perform_write_stall; + +/// Keyspace key (a.k.a. column family, locality group) +pub type KeyspaceKey = byteview::StrView; + +pub type InternalKeyspaceId = u64; + +pub fn apply_to_base_config( + config: lsm_tree::Config, + our_config: &CreateOptions, +) -> lsm_tree::Config { + config + // .level_count(our_config.level_count) + .data_block_size_policy(our_config.data_block_size_policy.clone()) + .data_block_compression_policy(our_config.data_block_compression_policy.clone()) + .index_block_compression_policy(our_config.index_block_compression_policy.clone()) + .data_block_restart_interval_policy(our_config.data_block_restart_interval_policy.clone()) + // .index_block_restart_interval_policy(our_config.index_block_restart_interval_policy.clone()) + .filter_block_pinning_policy(our_config.filter_block_pinning_policy.clone()) + .index_block_pinning_policy(our_config.index_block_pinning_policy.clone()) + .data_block_hash_ratio_policy(our_config.data_block_hash_ratio_policy.clone()) + .expect_point_read_hits(our_config.expect_point_read_hits) + .with_kv_separation(our_config.kv_separation_opts.clone()) + .index_block_partitioning_policy(our_config.index_block_partitioning_policy.clone()) + .filter_block_partitioning_policy(our_config.filter_block_partitioning_policy.clone()) + .filter_policy(our_config.filter_policy.clone()) + .with_compaction_filter_factory(our_config.compaction_filter_factory.clone()) +} + +pub struct KeyspaceInner { + /// Internal ID + pub(crate) id: InternalKeyspaceId, + + // Internal + // + /// Keyspace name + pub(crate) name: KeyspaceKey, + + // Keyspace configuration + #[doc(hidden)] + pub config: CreateOptions, + + /// If `true`, the keyspace is marked as deleted + pub(crate) is_deleted: AtomicBool, + + /// If `true`, fsync failed during persisting, see `Error::Poisoned` + pub(crate) is_poisoned: PoisonSignal, + + /// LSM-tree wrapper + #[doc(hidden)] + pub tree: AnyTree, + + pub(crate) supervisor: Supervisor, + + pub(crate) worker_messager: flume::Sender, + + #[expect(unused)] + lock_file: LockedFileGuard, +} + +impl Drop for KeyspaceInner { + fn drop(&mut self) { + log::trace!("Dropping KeyspaceInner: {:?}", self.name); + + if self.is_deleted.load(std::sync::atomic::Ordering::Acquire) { + let path = &self.tree.tree_config().path; + + // IMPORTANT: First, delete the manifest, + // once that is deleted, the keyspace is treated as uninitialized + // even if the .deleted marker is removed + // + // This is important, because if somehow `remove_dir_all` ends up + // deleting the `.deleted` marker first, we would end up resurrecting + // the keyspace + let manifest_file = path.join(LSM_CURRENT_VERSION_MARKER); + + // TODO: use https://github.com/rust-lang/rust/issues/31436 if stable + #[expect(clippy::collapsible_else_if)] + match manifest_file.try_exists() { + Ok(exists) => { + if exists { + if let Err(e) = std::fs::remove_file(manifest_file) { + log::error!( + "Failed to cleanup keyspace manifest at {}: {e}", + path.display(), + ); + } else { + if let Err(e) = std::fs::remove_dir_all(path) { + log::error!( + "Failed to cleanup deleted keyspace's folder at {}: {e}", + path.display(), + ); + } + } + } + } + Err(e) => { + log::error!( + "Failed to cleanup keyspace manifest at {}: {e}", + path.display(), + ); + } + } + } + + #[cfg(feature = "__internal_whitebox")] + crate::drop::decrement_drop_counter(); + } +} + +/// Handle to a keyspace +/// +/// Each keyspace is backed by an LSM-tree to provide a +/// disk-backed search tree, and can be configured individually. +/// +/// A keyspace generally only takes a little bit of memory and disk space, +/// but does not spawn its own background threads. +/// +/// As long as a handle to a keyspace is held, its folder is not cleaned up from disk +/// in case it is deleted from another thread. +#[derive(Clone)] +#[doc(alias = "column family")] +#[doc(alias = "locality group")] +#[doc(alias = "table")] +pub struct Keyspace(pub(crate) Arc); + +impl AsRef for &Keyspace { + fn as_ref(&self) -> &Keyspace { + self + } +} + +impl std::ops::Deref for Keyspace { + type Target = KeyspaceInner; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PartialEq for Keyspace { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for Keyspace {} + +impl std::hash::Hash for Keyspace { + fn hash(&self, state: &mut H) { + state.write(self.name.as_bytes()); + } +} + +impl Keyspace { + #[inline] + fn standard_tree(&self) -> &lsm_tree::Tree { + let lsm_tree::AnyTree::Standard(tree) = &self.tree else { + panic!("standard keyspace operation used with a blob keyspace"); + }; + tree + } + + #[doc(hidden)] + #[must_use] + pub fn id(&self) -> InternalKeyspaceId { + self.id + } + + /// Returns the keyspace's name. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert_eq!("default", &**tree.name()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn name(&self) -> &KeyspaceKey { + &self.name + } + + /// Clears the entire keyspace in O(1) time. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "a")?; + /// assert!(tree.contains_key("a")?); + /// + /// tree.clear()?; + /// assert!(!tree.contains_key("a")?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn clear(&self) -> crate::Result<()> { + let mut journal_writer = self.supervisor.journal.get_writer()?; + + // IMPORTANT: Check the poisoned flag after getting journal mutex, otherwise TOCTOU + if self.is_poisoned.is_poisoned() { + return Err(crate::Error::Poisoned); + } + + let seqno = self.supervisor.seqno.next(); + + journal_writer.write_clear(self.id, seqno)?; + + if !self.config.manual_journal_persist { + journal_writer + .persist(crate::PersistMode::Buffer) + .map_err(|e| { + log::error!("persist failed, which is a FATAL, and possibly hardware-related, failure: {e:?}"); + self.is_poisoned.poison(); + e + })?; + } + + self.tree.clear().inspect_err(|_| { + self.is_poisoned.poison(); + })?; + + self.supervisor.snapshot_tracker.publish(seqno); + + drop(journal_writer); + + Ok(()) + } + + /// Returns the number of blob bytes on disk that are not referenced. + /// + /// These will be reclaimed over time by blob garbage collection automatically. + #[must_use] + pub fn fragmented_blob_bytes(&self) -> u64 { + self.tree.stale_blob_bytes() + } + + #[doc(hidden)] + #[must_use] + pub fn sealed_memtable_count(&self) -> usize { + self.tree.sealed_memtable_count() + } + + /// Prepare ingestiom of a pre-sorted stream of key-value pairs into the keyspace. + /// + /// Prefer this method over singular inserts or write batches/transactions + /// for maximum bulk load speed. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + /// + /// # Panics + /// + /// Panics if the input iterator is not sorted in ascending order. + pub fn start_ingestion(&self) -> crate::Result> { + Ingestion::new(self) + } + + pub(crate) fn from_database( + keyspace_id: InternalKeyspaceId, + db: &Database, + tree: AnyTree, + name: KeyspaceKey, + config: CreateOptions, + ) -> Self { + Self(Arc::new(KeyspaceInner { + id: keyspace_id, + name, + tree, + config, + supervisor: db.supervisor.clone(), + worker_messager: db.worker_pool.sender.clone(), + is_deleted: AtomicBool::default(), + is_poisoned: db.is_poisoned.clone(), + lock_file: db.lock_file.clone(), + })) + } + + /// Creates a new keyspace. + pub(crate) fn create_new( + keyspace_id: InternalKeyspaceId, + db: &Database, + name: KeyspaceKey, + config: CreateOptions, + ) -> crate::Result { + log::debug!("Creating keyspace {name:?}->{keyspace_id}"); + + let base_folder = db + .config + .path + .join(KEYSPACES_FOLDER) + .join(keyspace_id.to_string()); + + std::fs::create_dir_all(&base_folder)?; + + let base_config = lsm_tree::Config::new( + base_folder, + db.supervisor.seqno.clone(), + db.supervisor.snapshot_tracker.get_ref(), + ) + .use_descriptor_table(db.config.descriptor_table.clone()) + .use_cache(db.config.cache.clone()); + + let base_config = apply_to_base_config(base_config, &config); + let tree = base_config.open()?; + + Ok(Self(Arc::new(KeyspaceInner { + supervisor: db.supervisor.clone(), + worker_messager: db.worker_pool.sender.clone(), + id: keyspace_id, + name, + config, + tree, + is_deleted: AtomicBool::default(), + is_poisoned: db.is_poisoned.clone(), + lock_file: db.lock_file.clone(), + }))) + } + + /// Returns the metrics struct of the underlying LSM-tree. + /// + /// # Note + /// + /// This function is experimental and metric names may change in future releases. + #[cfg(feature = "metrics")] + #[doc(hidden)] + pub fn metrics(&self) -> &lsm_tree::Metrics { + &**self.tree.metrics() + } + + /// Returns the underlying LSM-tree's path. + #[must_use] + pub fn path(&self) -> &Path { + self.tree.tree_config().path.as_path() + } + + /// Returns the disk space usage of this keyspace. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert_eq!(0, tree.disk_space()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn disk_space(&self) -> u64 { + self.tree.disk_space() + } + + /// Returns an iterator that scans through the entire keyspace. + /// + /// Avoid using this function, or limit it as otherwise it may scan a lot of items. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// tree.insert("f", "abc")?; + /// tree.insert("g", "abc")?; + /// assert_eq!(3, tree.iter().count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + #[expect(clippy::iter_without_into_iter)] + pub fn iter(&self) -> Iter { + let nonce = self.supervisor.snapshot_tracker.open(); + let iter = self.tree.iter(nonce.instant, None); + crate::iter::Iter::new(nonce, iter) + } + + #[doc(hidden)] + #[must_use] + pub fn iter_standard( + &self, + ) -> impl DoubleEndedIterator> + Send + 'static { + let nonce = self.supervisor.snapshot_tracker.open(); + let range = ..; + self.standard_tree() + .create_range::<&[u8], _>(&range, nonce.instant, None) + .map(move |item| { + let _keep_snapshot_alive = &nonce; + item.map_err(Into::into) + }) + } + + /// Returns an iterator over a range of items. + /// + /// Avoid using full or unbounded ranges as they may scan a lot of items (unless limited). + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// tree.insert("f", "abc")?; + /// tree.insert("g", "abc")?; + /// assert_eq!(2, tree.range("a"..="f").count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn range, R: RangeBounds>(&self, range: R) -> Iter { + let nonce = self.supervisor.snapshot_tracker.open(); + let iter = self.tree.range(range, SeqNo::MAX, None); + crate::iter::Iter::new(nonce, iter) + } + + #[doc(hidden)] + #[must_use] + pub fn range_standard, R: RangeBounds>( + &self, + range: R, + ) -> impl DoubleEndedIterator> + Send + 'static { + let nonce = self.supervisor.snapshot_tracker.open(); + self.standard_tree() + .create_range(&range, nonce.instant, None) + .map(move |item| { + let _keep_snapshot_alive = &nonce; + item.map_err(Into::into) + }) + } + + /// Returns an iterator over a prefixed set of items. + /// + /// Avoid using an empty prefix as it may scan a lot of items (unless limited). + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// tree.insert("ab", "abc")?; + /// tree.insert("abc", "abc")?; + /// assert_eq!(2, tree.prefix("ab").count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn prefix>(&self, prefix: K) -> Iter { + let nonce = self.supervisor.snapshot_tracker.open(); + let iter = self.tree.prefix(prefix, SeqNo::MAX, None); + crate::iter::Iter::new(nonce, iter) + } + + #[doc(hidden)] + #[must_use] + pub fn prefix_standard>( + &self, + prefix: K, + ) -> impl DoubleEndedIterator> + Send + 'static { + let nonce = self.supervisor.snapshot_tracker.open(); + self.standard_tree() + .create_prefix(prefix, nonce.instant, None) + .map(move |item| { + let _keep_snapshot_alive = &nonce; + item.map_err(Into::into) + }) + } + + /// Approximates the number of items in the keyspace. + /// + /// For update- or delete-heavy workloads, this value will + /// diverge from the real value, but is a O(1) operation. + /// + /// For insert-only workloads (e.g. logs, time series) + /// this value is reliable. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert_eq!(tree.approximate_len(), 0); + /// + /// tree.insert("1", "abc")?; + /// assert_eq!(tree.approximate_len(), 1); + /// + /// tree.remove("1")?; + /// // Oops! approximate_len will not be reliable here + /// assert_eq!(tree.approximate_len(), 2); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn approximate_len(&self) -> usize { + self.tree.approximate_len() + } + + /// Scans the entire keyspace, returning the number of items. + /// + /// # Caution + /// + /// This operation scans the entire keyspace: O(n) complexity! + /// + /// Never, under any circumstances, use .`len()` == 0 to check + /// if the keyspace is empty, use [`Keyspace::is_empty`] instead. + /// + /// If you want an estimate, use [`Keyspace::approximate_len`] instead. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert_eq!(tree.len()?, 0); + /// + /// tree.insert("1", "abc")?; + /// tree.insert("3", "abc")?; + /// tree.insert("5", "abc")?; + /// assert_eq!(tree.len()?, 3); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn len(&self) -> crate::Result { + let mut count = 0; + + for guard in self.iter() { + let _ = guard.key()?; + count += 1; + } + + Ok(count) + } + + /// Returns `true` if the keyspace is empty. + /// + /// This operation has O(log N) complexity. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert!(tree.is_empty()?); + /// + /// tree.insert("a", "abc")?; + /// assert!(!tree.is_empty()?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn is_empty(&self) -> crate::Result { + self.tree.is_empty(SeqNo::MAX, None).map_err(Into::into) + } + + /// Returns `true` if the keyspace contains the specified key. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert!(!tree.contains_key("a")?); + /// + /// tree.insert("a", "abc")?; + /// assert!(tree.contains_key("a")?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn contains_key>(&self, key: K) -> crate::Result { + self.tree.contains_key(key, SeqNo::MAX).map_err(Into::into) + } + + /// Retrieves an item from the keyspace. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("my_value".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn get>(&self, key: K) -> crate::Result> { + Ok(self.tree.get(key, SeqNo::MAX)?) + } + + #[doc(hidden)] + pub fn get_standard>( + &self, + key: K, + ) -> crate::Result> { + Ok(self.standard_tree().get(key, SeqNo::MAX)?) + } + + /// Retrieves the size of an item from the keyspace. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let len = tree.size_of("a")?.unwrap_or_default(); + /// assert_eq!("my_value".len() as u32, len); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn size_of>(&self, key: K) -> crate::Result> { + Ok(self.tree.size_of(key, SeqNo::MAX)?) + } + + /// Returns the first key-value pair in the keyspace. + /// The key in this pair is the minimum key in the keyspace. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("1", "abc")?; + /// tree.insert("3", "abc")?; + /// tree.insert("5", "abc")?; + /// + /// let key = tree.first_key_value().expect("item should exist").key()?; + /// assert_eq!(&*key, "1".as_bytes()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn first_key_value(&self) -> Option { + self.tree.first_key_value(SeqNo::MAX, None).map(Guard) + } + + /// Returns the last key-value pair in the keyspace. + /// The key in this pair is the maximum key in the keyspace. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("1", "abc")?; + /// tree.insert("3", "abc")?; + /// tree.insert("5", "abc")?; + /// + /// let key = tree.last_key_value().expect("item should exist").key()?; + /// assert_eq!(&*key, "5".as_bytes()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn last_key_value(&self) -> Option { + self.tree.last_key_value(SeqNo::MAX, None).map(Guard) + } + + /// Returns `true` if the underlying LSM-tree is key-value-separated. + #[must_use] + pub fn is_kv_separated(&self) -> bool { + matches!(self.tree, crate::AnyTree::Blob(_)) + } + + // NOTE: Used in tests + #[doc(hidden)] + pub fn rotate_memtable_and_wait(&self) -> crate::Result<()> { + if self.rotate_memtable()? { + self.supervisor.flush_manager.wait_for_empty(); + + while self.tree.sealed_memtable_count() > 0 { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + Ok(()) + } + + #[doc(hidden)] + pub fn rotate_memtable(&self) -> crate::Result { + log::trace!("acquiring journal lock"); + let journal_writer = self.supervisor.journal.get_writer()?; + let active_memtable_id = self.tree.active_memtable().id(); + self.inner_rotate_memtable(journal_writer, active_memtable_id) + } + + pub(crate) fn inner_rotate_memtable( + &self, + journal_writer: MutexGuard<'_, crate::journal::writer::Writer>, + memtable_id: lsm_tree::MemtableId, + ) -> crate::Result { + log::debug!("Rotating keyspace {:?}", self.name); + + if self.tree.active_memtable().id() != memtable_id { + return Ok(false); + } + + // Rotate memtable + let Some(_) = self.tree.rotate_memtable() else { + log::debug!("Got no sealed memtable, someone beat us to it"); + return Ok(false); + }; + + drop(journal_writer); + + self.supervisor.flush_manager.enqueue(Arc::new(FlushTask { + keyspace: self.clone(), + })); + + self.worker_messager.send(WorkerMessage::Flush).ok(); + + { + // NOTE: If the difference between watermark is too large, and + // we never opened a snapshot, we need to pull the watermark up + // + // https://github.com/fjall-rs/fjall/discussions/85 + self.supervisor.snapshot_tracker.pullup(); + self.supervisor.snapshot_tracker.gc(); + + for keyspace in self + .supervisor + .keyspaces + .read() + .expect("lock is poisoned") + .values() + { + if let Err(e) = keyspace.tree.get_version_history_lock().maintenance( + keyspace.path(), + self.supervisor.snapshot_tracker.get_seqno_safe_to_gc(), + ) { + log::warn!( + "Version history GC failed for keyspace {:?}: {e:?}", + keyspace.name, + ); + } + } + } + + #[expect(clippy::expect_used)] + self.supervisor + .journal_manager + .write() + .expect("lock is poisoned") + .maintenance()?; + + Ok(true) + } + + fn check_write_halt(&self) { + while self.tree.l0_run_count() >= 30 { + std::thread::sleep(Duration::from_millis(10)); + } + } + + pub(crate) fn local_backpressure(&self) -> bool { + let mut throttled = false; + + let l0_run_count = self.tree.l0_run_count(); + + if l0_run_count >= 20 { + perform_write_stall(l0_run_count); + self.check_write_halt(); + throttled = true; + } + + while self.tree.sealed_memtable_count() >= 4 { + log::debug!( + "Halting writes because we have 4+ sealed memtables in {:?} queued up", + self.name, + ); + std::thread::sleep(Duration::from_millis(100)); + throttled = true; + } + + throttled + } + + pub(crate) fn request_rotation(&self) { + let lock = self.tree.get_version_history_lock(); + let latest_version = lock.latest_version(); + let active_memtable = &latest_version.active_memtable; + + self.worker_messager + .try_send(WorkerMessage::RotateMemtable( + self.clone(), + active_memtable.id(), + )) + .ok(); + } + + pub(crate) fn check_memtable_rotate(&self, size: u64) { + if size > self.config.max_memtable_size { + self.request_rotation(); + } + } + + fn maintenance(&self, memtable_size: u64) { + self.check_memtable_rotate(memtable_size); + self.local_backpressure(); + } + + #[doc(hidden)] + #[must_use] + pub fn l0_table_count(&self) -> usize { + self.tree.level_table_count(0).unwrap_or_default() + } + + /// Number of tables (a.k.a. SST files) in the LSM-tree. + #[doc(hidden)] + #[must_use] + pub fn table_count(&self) -> usize { + self.tree.table_count() + } + + /// Number of blob files in the LSM-tree. + #[doc(hidden)] + #[must_use] + pub fn blob_file_count(&self) -> usize { + self.tree.blob_file_count() + } + + /// Performs major compaction, blocking the caller until it's done. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn major_compact(&self) -> crate::Result<()> { + self.tree.major_compact( + 64_000_000, + self.supervisor.snapshot_tracker.get_seqno_safe_to_gc(), + )?; + + // TODO: 3.0.0 ----^ + // compaction strategy needs a method: strategy.table_target_size() + + Ok(()) + } + + /// Inserts a key-value pair into the keyspace. + /// + /// Keys may be up to 65536 bytes long, values up to 2^32 bytes. + /// Shorter keys and values result in better performance. + /// + /// If the key already exists, the item will be overwritten. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// assert!(!tree.is_empty()?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn insert, V: Into>( + &self, + key: K, + value: V, + ) -> crate::Result<()> { + use std::sync::atomic::Ordering; + + if self.is_deleted.load(Ordering::Relaxed) { + return Err(crate::Error::KeyspaceDeleted); + } + + let key = key.into(); + let value = value.into(); + + let mut journal_writer = self.supervisor.journal.get_writer()?; + + // IMPORTANT: Check the poisoned flag after getting journal mutex, otherwise TOCTOU + if self.is_poisoned.is_poisoned() { + return Err(crate::Error::Poisoned); + } + + let seqno = self.supervisor.seqno.next(); + + journal_writer + .write_raw(self.id, &key, &value, lsm_tree::ValueType::Value, seqno) + .inspect_err(|_| { + self.is_poisoned.poison(); + })?; + + if !self.config.manual_journal_persist { + journal_writer + .persist(crate::PersistMode::Buffer) + .inspect_err(|e| { + log::error!("persist failed, which is a FATAL, and possibly hardware-related, failure: {e:?}"); + self.is_poisoned.poison(); + })?; + } + + let (item_size, memtable_size) = self.tree.insert(key, value, seqno); + + self.supervisor.snapshot_tracker.publish(seqno); + + drop(journal_writer); + + self.supervisor.write_buffer_size.allocate(item_size); + self.maintenance(memtable_size); + + Ok(()) + } + + /// Removes an item from the keyspace. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let item = tree.get("a")?.expect("should have item"); + /// assert_eq!("abc".as_bytes(), &*item); + /// + /// tree.remove("a")?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(None, item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn remove>(&self, key: K) -> crate::Result<()> { + use std::sync::atomic::Ordering; + + if self.is_deleted.load(Ordering::Relaxed) { + return Err(crate::Error::KeyspaceDeleted); + } + + let key = key.into(); + + let mut journal_writer = self.supervisor.journal.get_writer()?; + + // IMPORTANT: Check the poisoned flag after getting journal mutex, otherwise TOCTOU + if self.is_poisoned.is_poisoned() { + return Err(crate::Error::Poisoned); + } + + let seqno = self.supervisor.seqno.next(); + + journal_writer + .write_raw(self.id, &key, &[], lsm_tree::ValueType::Tombstone, seqno) + .inspect_err(|_| { + self.is_poisoned.poison(); + })?; + + if !self.config.manual_journal_persist { + journal_writer + .persist(crate::PersistMode::Buffer) + .inspect_err(|e| { + log::error!("persist failed, which is a FATAL, and possibly hardware-related, failure: {e:?}"); + self.is_poisoned.poison(); + })?; + } + + let (item_size, memtable_size) = self.tree.remove(key, seqno); + + self.supervisor.snapshot_tracker.publish(seqno); + + drop(journal_writer); + + self.supervisor.write_buffer_size.allocate(item_size); + self.maintenance(memtable_size); + + Ok(()) + } + + /// Removes an item from the keyspace, leaving behind a weak tombstone. + /// + /// When a weak tombstone is matched with a single write in a compaction, + /// the tombstone will be removed along with the value. If the key was + /// overwritten the result of a `remove_weak` is undefined. + /// + /// Only use this remove if it is known that the key has only been written + /// to once since its creation or last `remove_weak`. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// # Experimental + /// + /// This function is currently experimental. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let item = tree.get("a")?.expect("should have item"); + /// assert_eq!("abc".as_bytes(), &*item); + /// + /// tree.remove_weak("a")?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(None, item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn remove_weak>(&self, key: K) -> crate::Result<()> { + use std::sync::atomic::Ordering; + + if self.is_deleted.load(Ordering::Relaxed) { + return Err(crate::Error::KeyspaceDeleted); + } + + let key = key.into(); + + let mut journal_writer = self.supervisor.journal.get_writer()?; + + // IMPORTANT: Check the poisoned flag after getting journal mutex, otherwise TOCTOU + if self.is_poisoned.is_poisoned() { + return Err(crate::Error::Poisoned); + } + + let seqno = self.supervisor.seqno.next(); + + journal_writer + .write_raw( + self.id, + &key, + &[], + lsm_tree::ValueType::WeakTombstone, + seqno, + ) + .inspect_err(|_| { + self.is_poisoned.poison(); + })?; + + if !self.config.manual_journal_persist { + journal_writer + .persist(crate::PersistMode::Buffer) + .inspect_err(|e| { + log::error!( + "persist failed, which is a FATAL, and possibly hardware-related, failure: {e:?}" + ); + self.is_poisoned.poison(); + })?; + } + + let (item_size, memtable_size) = self.tree.remove(key, seqno); + + self.supervisor.snapshot_tracker.publish(seqno); + + drop(journal_writer); + + self.supervisor.write_buffer_size.allocate(item_size); + self.maintenance(memtable_size); + + Ok(()) + } +} diff --git a/crates/fjall/src/keyspace/name.rs b/crates/fjall/src/keyspace/name.rs new file mode 100644 index 000000000..107308965 --- /dev/null +++ b/crates/fjall/src/keyspace/name.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Keyspace names can be up to 255 characters long, can not be empty and +/// can only contain alphanumerics, underscore (`_`), dash (`-`), dot (`.`), hash tag (`#`) and dollar (`$`). +#[expect(clippy::module_name_repetitions)] +pub fn is_valid_keyspace_name(s: &str) -> bool { + if s.is_empty() { + return false; + } + + u8::try_from(s.len()).is_ok() +} diff --git a/crates/fjall/src/keyspace/options.rs b/crates/fjall/src/keyspace/options.rs new file mode 100644 index 000000000..1b8455aa3 --- /dev/null +++ b/crates/fjall/src/keyspace/options.rs @@ -0,0 +1,749 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + config::{ + BlockSizePolicy, BloomConstructionPolicy, CompressionPolicy, FilterPolicy, + FilterPolicyEntry, HashRatioPolicy, PartitioningPolicy, PinningPolicy, + RestartIntervalPolicy, + }, + keyspace::{config::DecodeConfig, InternalKeyspaceId}, + meta_keyspace::{encode_config_key, MetaKeyspace}, +}; +use byteorder::ReadBytesExt; +use lsm_tree::{ + compaction::Factory as CompactionFilterFactory, CompressionType, KvPair, KvSeparationOptions, +}; +use std::sync::Arc; + +/// Options to configure a keyspace +#[expect(clippy::module_name_repetitions)] +#[derive(Clone)] +pub struct CreateOptions { + /// Number of levels of the LSM tree (depth of tree). + pub(crate) level_count: u8, + + /// Maximum size of this keyspace's memtable - can be changed during runtime + pub(crate) max_memtable_size: u64, + + /// Data block hash ratio + #[doc(hidden)] + pub data_block_hash_ratio_policy: HashRatioPolicy, + + /// Block size of data blocks. + #[doc(hidden)] + pub data_block_size_policy: BlockSizePolicy, + + #[doc(hidden)] + pub data_block_restart_interval_policy: RestartIntervalPolicy, + + #[doc(hidden)] + pub index_block_restart_interval_policy: RestartIntervalPolicy, + + #[doc(hidden)] + pub index_block_pinning_policy: PinningPolicy, + + #[doc(hidden)] + pub filter_block_pinning_policy: PinningPolicy, + + #[doc(hidden)] + pub filter_block_partitioning_policy: PartitioningPolicy, + + #[doc(hidden)] + pub index_block_partitioning_policy: PartitioningPolicy, + + /// If `true`, the last level will not build filters, reducing the filter size of a database + /// by ~90% typically. + #[doc(hidden)] + pub expect_point_read_hits: bool, + + /// Filter construction policy. + #[doc(hidden)] + pub filter_policy: FilterPolicy, + + /// Compression to use for data blocks. + #[doc(hidden)] + pub data_block_compression_policy: CompressionPolicy, + + /// Compression to use for index blocks. + #[doc(hidden)] + pub index_block_compression_policy: CompressionPolicy, + + pub(crate) manual_journal_persist: bool, + + #[doc(hidden)] + pub compaction_strategy: Arc, + + #[doc(hidden)] + pub kv_separation_opts: Option, + + pub(crate) compaction_filter_factory: Option>, +} + +impl Default for CreateOptions { + fn default() -> Self { + let default_tree_config = lsm_tree::Config::default(); + + Self { + manual_journal_persist: false, + + max_memtable_size: /* 64 MiB */ 64 * 1_024 * 1_024, + + data_block_hash_ratio_policy: HashRatioPolicy::all(0.0), + + data_block_size_policy: BlockSizePolicy::all(/* 4 KiB */ 4 * 1_024), + + data_block_restart_interval_policy: RestartIntervalPolicy::new([10, 16]), + index_block_restart_interval_policy: RestartIntervalPolicy::all(1), + + index_block_pinning_policy: PinningPolicy::new([true, true, false]), + filter_block_pinning_policy: PinningPolicy::new([true, false]), + + index_block_partitioning_policy: PartitioningPolicy::new([false, false, false, true]), + filter_block_partitioning_policy: PartitioningPolicy::new([false, false, false, true]), + + expect_point_read_hits: false, + + filter_policy: FilterPolicy::new([ + FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.0001)), + FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)), + ]), + + level_count: default_tree_config.level_count, + + #[cfg(feature = "lz4")] + data_block_compression_policy: CompressionPolicy::new([CompressionType::None, CompressionType::None, CompressionType::Lz4]), + + #[cfg(not(feature = "lz4"))] + data_block_compression_policy: CompressionPolicy::new(&[CompressionType::None]), + + index_block_compression_policy: CompressionPolicy::all(CompressionType::None), + + compaction_strategy: Arc::new( + crate::compaction::Leveled::default() + ), + + kv_separation_opts: None, + + compaction_filter_factory: None, + } + } +} + +macro_rules! policy { + ($keyspace_id:expr, $name:expr, $field:expr) => {{ + let key = encode_config_key($keyspace_id, $name); + (key.into(), $field.encode()) + }}; +} + +impl CreateOptions { + /// Installs a compaction filter factory. + pub(crate) fn with_compaction_filter_factory( + mut self, + factory: Arc, + ) -> Self { + self.compaction_filter_factory = Some(factory); + self + } + + #[expect(clippy::expect_used, clippy::too_many_lines)] + pub(crate) fn from_kvs( + keyspace_id: InternalKeyspaceId, + meta_keyspace: &MetaKeyspace, + ) -> crate::Result { + let blob = meta_keyspace.get_kv_for_config(keyspace_id, "blob")?; + + let data_block_compression_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "data_block_compression_policy")? + .expect("should exist"); + let data_block_compression_policy = + CompressionPolicy::decode(&data_block_compression_policy)?; + + let index_block_compression_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "index_block_compression_policy")? + .expect("should exist"); + let index_block_compression_policy = + CompressionPolicy::decode(&index_block_compression_policy)?; + + let data_block_size_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "data_block_size_policy")? + .expect("should exist"); + let data_block_size_policy = BlockSizePolicy::decode(&data_block_size_policy)?; + + let filter_block_partitioning_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "filter_block_partitioning_policy")? + .expect("should exist"); + let filter_block_partitioning_policy = + PinningPolicy::decode(&filter_block_partitioning_policy)?; + + let index_block_partitioning_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "index_block_partitioning_policy")? + .expect("should exist"); + let index_block_partitioning_policy = + PinningPolicy::decode(&index_block_partitioning_policy)?; + + let filter_block_pinning_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "filter_block_pinning_policy")? + .expect("should exist"); + let filter_block_pinning_policy = PinningPolicy::decode(&filter_block_pinning_policy)?; + + let index_block_pinning_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "index_block_pinning_policy")? + .expect("should exist"); + let index_block_pinning_policy = PinningPolicy::decode(&index_block_pinning_policy)?; + + let data_block_restart_interval_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "data_block_restart_interval_policy")? + .expect("should exist"); + let data_block_restart_interval_policy = + RestartIntervalPolicy::decode(&data_block_restart_interval_policy)?; + + let index_block_restart_interval_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "index_block_restart_interval_policy")? + .expect("should exist"); + let index_block_restart_interval_policy = + RestartIntervalPolicy::decode(&index_block_restart_interval_policy)?; + + let data_block_hash_ratio_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "data_block_hash_ratio_policy")? + .expect("should exist"); + let data_block_hash_ratio_policy = HashRatioPolicy::decode(&data_block_hash_ratio_policy)?; + + let expect_point_read_hits = meta_keyspace + .get_kv_for_config(keyspace_id, "expect_point_read_hits")? + .expect("should exist"); + let expect_point_read_hits = expect_point_read_hits == [1]; + + let filter_policy = meta_keyspace + .get_kv_for_config(keyspace_id, "filter_policy")? + .expect("should exist"); + let filter_policy = FilterPolicy::decode(&filter_policy)?; + + let blob_opts = blob.map(|_| { + use byteorder::LE; + use lsm_tree::coding::Decode; + + let blob_age_cutoff = meta_keyspace + .get_kv_for_config(keyspace_id, "blob_age_cutoff")? + .expect("blob_age_cutoff should be defined"); + let blob_age_cutoff = (&mut &blob_age_cutoff[..]).read_f32::()?; + + let blob_compression = meta_keyspace + .get_kv_for_config(keyspace_id, "blob_compression")? + .expect("blob_compression should be defined"); + let blob_compression = CompressionType::decode_from(&mut &blob_compression[..])?; + + let file_target_size = meta_keyspace + .get_kv_for_config(keyspace_id, "blob_file_target_size")? + .expect("blob_file_target_size should be defined"); + let file_target_size = (&mut &file_target_size[..]).read_u64::()?; + + let separation_threshold = meta_keyspace + .get_kv_for_config(keyspace_id, "blob_separation_threshold")? + .expect("blob_separation_threshold should be defined"); + let separation_threshold = (&mut &separation_threshold[..]).read_u32::()?; + + let staleness_threshold = meta_keyspace + .get_kv_for_config(keyspace_id, "blob_staleness_threshold")? + .expect("blob_staleness_threshold should be defined"); + let staleness_threshold = (&mut &staleness_threshold[..]).read_f32::()?; + + Ok::<_, crate::Error>( + KvSeparationOptions::default() + .compression(blob_compression) + .file_target_size(file_target_size) + .separation_threshold(separation_threshold) + .staleness_threshold(staleness_threshold) + .age_cutoff(blob_age_cutoff), + ) + }); + + let compaction_strategy_name = meta_keyspace + .get_kv_for_config(keyspace_id, "compaction_strategy")? + .expect("compaction_strategy should be defined"); + + let compaction_strategy_name = std::str::from_utf8(&compaction_strategy_name) + .expect("compaction_strategy should be UTF-8"); + + let compaction_strategy = match compaction_strategy_name { + lsm_tree::compaction::LEVELED_COMPACTION_NAME => { + use byteorder::LE; + + let l0_threshold = meta_keyspace + .get_kv_for_config(keyspace_id, "leveled_l0_threshold")? + .expect("leveled_l0_threshold should be defined"); + let l0_threshold = (&mut &l0_threshold[..]).read_u8()?; + + let target_size = meta_keyspace + .get_kv_for_config(keyspace_id, "leveled_target_size")? + .expect("leveled_target_size should be defined"); + let target_size = (&mut &target_size[..]).read_u64::()?; + + let level_ratio_policy_bytes = meta_keyspace + .get_kv_for_config(keyspace_id, "leveled_level_ratio_policy")? + .expect("leveled_level_ratio_policy should be defined"); + let level_ratio_policy_bytes = &mut &level_ratio_policy_bytes[..]; + + let level_ratio_policy_len = level_ratio_policy_bytes.read_u8()?; + + let mut ratios = vec![]; + + for _ in 0..level_ratio_policy_len { + ratios.push(level_ratio_policy_bytes.read_f32::()?); + } + + Arc::new( + crate::compaction::Leveled::default() + .with_l0_threshold(l0_threshold) + .with_table_target_size(target_size) + .with_level_ratio_policy(ratios), + ) as Arc + } + lsm_tree::compaction::FIFO_COMPACTION_NAME => { + use byteorder::LE; + + let fifo_limit = meta_keyspace + .get_kv_for_config(keyspace_id, "fifo_limit")? + .expect("fifo_limit should be defined"); + let fifo_limit = (&mut &fifo_limit[..]).read_u64::()?; + + let has_ttl = meta_keyspace + .get_kv_for_config(keyspace_id, "fifo_ttl")? + .expect("fifo_ttl should be defined") + == [1]; + + let ttl_seconds = if has_ttl { + let fifo_ttl_seconds = meta_keyspace + .get_kv_for_config(keyspace_id, "fifo_ttl_seconds")? + .expect("fifo_ttl_seconds should be defined"); + let fifo_ttl_seconds = (&mut &fifo_ttl_seconds[..]).read_u64::()?; + + Some(fifo_ttl_seconds) + } else { + None + }; + + Arc::new(crate::compaction::Fifo::new(fifo_limit, ttl_seconds)) + } + name => { + panic!("Invalid/unsupported compaction strategy: {name:?}"); + } + }; + + let manual_journal_persist = meta_keyspace + .get_kv_for_config(keyspace_id, "manual_journal_persist")? + .expect("should exist") + == [1]; + + let max_memtable_size = meta_keyspace + .get_kv_for_config(keyspace_id, "max_memtable_size")? + .expect("should exist"); + let max_memtable_size = (&mut &max_memtable_size[..]).read_u64::()?; + + Ok(Self { + data_block_hash_ratio_policy, + + filter_block_partitioning_policy, + index_block_partitioning_policy, + + filter_block_pinning_policy, + index_block_pinning_policy, + + data_block_compression_policy, + index_block_compression_policy, + + data_block_size_policy, + + data_block_restart_interval_policy, + index_block_restart_interval_policy, + + expect_point_read_hits, + filter_policy, + + level_count: 7, // Levels are currently hard coded to 7 + + manual_journal_persist, + + max_memtable_size, + + compaction_strategy, + + kv_separation_opts: blob_opts.transpose()?, + + compaction_filter_factory: None, + }) + } + + #[expect(clippy::too_many_lines)] + pub(crate) fn encode_kvs(&self, keyspace_id: InternalKeyspaceId) -> Vec { + use crate::keyspace::config::EncodeConfig; + + let mut kvs = vec![ + { + let key = encode_config_key(keyspace_id, "compaction_strategy"); + (key, self.compaction_strategy.get_name().into()) + }, + policy!( + keyspace_id, + "data_block_compression_policy", + self.data_block_compression_policy + ), + policy!( + keyspace_id, + "data_block_hash_ratio_policy", + self.data_block_hash_ratio_policy + ), + policy!( + keyspace_id, + "data_block_restart_interval_policy", + self.data_block_restart_interval_policy + ), + policy!( + keyspace_id, + "data_block_size_policy", + self.data_block_size_policy + ), + { + let key = encode_config_key(keyspace_id, "expect_point_read_hits"); + + let value = (if self.expect_point_read_hits { + [1u8] + } else { + [0u8] + }) + .into(); + + (key, value) + }, + policy!( + keyspace_id, + "filter_block_partitioning_policy", + self.filter_block_partitioning_policy + ), + policy!( + keyspace_id, + "filter_block_pinning_policy", + self.filter_block_pinning_policy + ), + policy!(keyspace_id, "filter_policy", self.filter_policy), + policy!( + keyspace_id, + "index_block_compression_policy", + self.index_block_compression_policy + ), + policy!( + keyspace_id, + "index_block_partitioning_policy", + self.index_block_partitioning_policy + ), + policy!( + keyspace_id, + "index_block_pinning_policy", + self.index_block_pinning_policy + ), + policy!( + keyspace_id, + "index_block_restart_interval_policy", + self.index_block_restart_interval_policy + ), + { + let key = encode_config_key(keyspace_id, "level_count"); + (key, [self.level_count].into()) + }, + { + let key = encode_config_key(keyspace_id, "manual_journal_persist"); + (key, [u8::from(self.manual_journal_persist)].into()) + }, + { + let key = encode_config_key(keyspace_id, "max_memtable_size"); + (key, self.max_memtable_size.to_le_bytes().into()) + }, + { + let key = encode_config_key(keyspace_id, "version"); + (key, [3u8].into()) + }, + ]; + + match self.compaction_strategy.get_name() { + "LeveledCompaction" | "FifoCompaction" => { + kvs.extend( + self.compaction_strategy + .get_config() + .into_iter() + .map(|(k, v)| { + // TODO: this is a bit stupid right now because we depend on behaviour in lsm-tree + // we should probably make lsm-tree just return a String for the key + let k = std::str::from_utf8(&k) + .expect("compaction strategy should return UTF-8 key"); + + (encode_config_key(keyspace_id, k), v) + }), + ); + } + name => { + panic!("Invalid/unsupported compaction strategy: {name:?}"); + } + } + + if let Some(blob_opts) = &self.kv_separation_opts { + kvs.extend([ + { + let key = encode_config_key(keyspace_id, "blob"); + (key, [1u8].into()) + }, + { + let key = encode_config_key(keyspace_id, "blob_age_cutoff"); + (key, blob_opts.age_cutoff.to_le_bytes().into()) + }, + { + use lsm_tree::coding::Encode; + + let key = encode_config_key(keyspace_id, "blob_compression"); + (key, blob_opts.compression.encode_into_vec().into()) + }, + { + let key = encode_config_key(keyspace_id, "blob_file_target_size"); + (key, blob_opts.file_target_size.to_le_bytes().into()) + }, + { + let key = encode_config_key(keyspace_id, "blob_separation_threshold"); + (key, blob_opts.separation_threshold.to_le_bytes().into()) + }, + { + let key = encode_config_key(keyspace_id, "blob_staleness_threshold"); + (key, blob_opts.staleness_threshold.to_le_bytes().into()) + }, + ]); + } + + kvs + } + + /// Toggles key-value separation. + #[must_use] + pub fn with_kv_separation(mut self, opts: Option) -> Self { + self.kv_separation_opts = opts; + self + } + + /// Sets the restart interval inside data blocks. + /// + /// A higher restart interval saves space while increasing lookup times + /// inside data blocks. + /// + /// Default = 16 + #[must_use] + pub fn data_block_restart_interval_policy(mut self, policy: RestartIntervalPolicy) -> Self { + self.data_block_restart_interval_policy = policy; + self + } + + // TODO: not supported yet in lsm-tree + // /// Sets the restart interval inside index blocks. + // /// + // /// A higher restart interval saves space while increasing lookup times + // /// inside index blocks. + // /// + // /// Default = 1 + // #[must_use] + // #[doc(hidden)] + // pub fn index_block_restart_interval_policy(mut self, policy: RestartIntervalPolicy) -> Self { + // self.index_block_restart_interval_policy = policy; + // self + // } + + /// Sets the pinning policy for filter blocks. + /// + /// By default, L0 filter blocks are pinned. + #[must_use] + pub fn filter_block_pinning_policy(mut self, policy: PinningPolicy) -> Self { + self.filter_block_pinning_policy = policy; + self + } + + /// Sets the pinning policy for index blocks. + /// + /// By default, L0 and L1 index blocks are pinned. + #[must_use] + pub fn index_block_pinning_policy(mut self, policy: PinningPolicy) -> Self { + self.index_block_pinning_policy = policy; + self + } + + /// Sets the partitioning policy for filter blocks. + #[must_use] + pub fn filter_block_partitioning_policy(mut self, policy: PartitioningPolicy) -> Self { + self.filter_block_partitioning_policy = policy; + self + } + + /// Sets the partitioning policy for index blocks. + #[must_use] + pub fn index_block_partitioning_policy(mut self, policy: PartitioningPolicy) -> Self { + self.index_block_partitioning_policy = policy; + self + } + + /// Sets the hash ratio for the hash index in data blocks. + /// + /// The hash index speeds up point queries by using an embedded + /// hash map in data blocks, but uses more space/memory. + /// + /// In-memory or heavily cached workloads benefit more from a higher hash ratio. + /// + /// If 0.0, the hash index is not constructed. + #[must_use] + #[doc(hidden)] + pub fn data_block_hash_ratio_policy(mut self, policy: HashRatioPolicy) -> Self { + self.data_block_hash_ratio_policy = policy; + self + } + + /// Sets the filter policy for data blocks. + #[must_use] + pub fn filter_policy(mut self, policy: FilterPolicy) -> Self { + self.filter_policy = policy; + self + } + + /// If `true`, the last level will not build filters, reducing the filter size of a database + /// by ~90% typically. + /// + /// **Enable this only if you know that point reads generally are expected to find a key-value pair.** + #[must_use] + pub fn expect_point_read_hits(mut self, b: bool) -> Self { + self.expect_point_read_hits = b; + self + } + + /// Sets the compression policy for data blocks. + #[must_use] + pub fn data_block_compression_policy(mut self, policy: CompressionPolicy) -> Self { + self.data_block_compression_policy = policy; + self + } + + /// Sets the compression policy for index blocks. + #[must_use] + pub fn index_block_compression_policy(mut self, policy: CompressionPolicy) -> Self { + self.index_block_compression_policy = policy; + self + } + + /// Sets the compaction strategy. + /// + /// Default = Leveled + #[must_use] + pub fn compaction_strategy( + mut self, + compaction_strategy: Arc, + ) -> Self { + self.compaction_strategy = compaction_strategy; + self + } + + /// If `false`, writes will flush data to the operating system. + /// + /// Default = false + /// + /// Set to `true` to handle persistence manually, e.g. manually using `PersistMode::SyncData`. + #[must_use] + pub fn manual_journal_persist(mut self, flag: bool) -> Self { + self.manual_journal_persist = flag; + self + } + + /// Sets the maximum memtable size. + /// + /// Default = 64 MiB + /// + /// Recommended size 8 - 64 MiB, depending on how much memory + /// is available. + /// + /// Conversely, if `max_memtable_size` is larger than 64 MiB, + /// it may require increasing the database's `max_write_buffer_size`. + #[must_use] + pub fn max_memtable_size(mut self, bytes: u64) -> Self { + self.max_memtable_size = bytes; + self + } + + /// Sets the block size. + /// + /// Once set for a keyspace, this property is not considered in the future. + /// + /// Default = 4 KiB + /// + /// For point read heavy workloads (get) a sensible default is + /// somewhere between 4 - 8 KiB, depending on the average value size. + /// + /// For more space efficiency, block size between 16 - 64 KiB are sensible. + /// + /// # Panics + /// + /// Panics if the block size is smaller than 1 KiB or larger than 1 MiB. + #[must_use] + pub fn data_block_size_policy(mut self, policy: BlockSizePolicy) -> Self { + self.data_block_size_policy = policy; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + #[cfg(not(feature = "lz4"))] + fn keyspace_opts_compression_none() { + let mut c = CreateOptions::default(); + assert_eq!( + c.data_block_compression_policy, + CompressionPolicy::disabled(), + ); + assert_eq!(c.kv_separation_opts, None); + + c = c.with_kv_separation(KvSeparationOptions::default()); + assert_eq!( + c.kv_separation_opts.as_ref().unwrap().compression, + CompressionType::None, + ); + + c = c.data_block_compression_policy(CompressionPolicy::disabled()); + assert_eq!( + c.data_block_compression_policy, + CompressionPolicy::disabled(), + ); + assert_eq!( + c.kv_separation_opts.unwrap().compression, + CompressionType::None, + ); + } + + #[test] + #[expect(clippy::unwrap_used)] + #[cfg(feature = "lz4")] + fn keyspace_opts_compression_default() { + use CompressionType::{Lz4, None as Uncompressed}; + + let mut c = CreateOptions::default(); + assert_eq!( + c.data_block_compression_policy, + CompressionPolicy::new([Uncompressed, Uncompressed, Lz4]), + ); + assert_eq!(c.kv_separation_opts, None); + + c = c.with_kv_separation(Some(KvSeparationOptions::default())); + assert_eq!(c.kv_separation_opts.as_ref().unwrap().compression, Lz4); + + c = c.data_block_compression_policy(CompressionPolicy::disabled()); + assert_eq!( + c.data_block_compression_policy, + CompressionPolicy::disabled(), + ); + assert_eq!(c.kv_separation_opts.as_ref().unwrap().compression, Lz4); + } +} diff --git a/crates/fjall/src/keyspace/test.rs b/crates/fjall/src/keyspace/test.rs new file mode 100644 index 000000000..64a8cb2cd --- /dev/null +++ b/crates/fjall/src/keyspace/test.rs @@ -0,0 +1,75 @@ +use crate::{Database, KeyspaceCreateOptions, UserKey, UserValue}; + +#[test_log::test] +#[ignore = "flimsy because of the compaction check, probably race condition... run the compaction synchronously"] +fn keyspace_ingest() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).worker_threads(0).open()?; + let items = db.keyspace("items", KeyspaceCreateOptions::default)?; + + { + let mut ingest = items.start_ingestion()?; + + for (k, v) in [0u8, 1, 2, 3, 4, 5] + .into_iter() + .map(|i| (UserKey::new(&i.to_be_bytes()), UserValue::empty())) + { + ingest.write(k, v)?; + } + + ingest.finish()?; + }; + assert_eq!(6, items.len()?); + assert_eq!(1, items.table_count()); + + { + let mut ingest = items.start_ingestion()?; + + for (k, v) in [1u8, 6, 7, 8, 9] + .into_iter() + .map(|i| (UserKey::new(&i.to_be_bytes()), UserValue::empty())) + { + ingest.write(k, v)?; + } + + ingest.finish()?; + } + assert_eq!(10, items.len()?); + assert_eq!(2, items.table_count()); + + { + let mut ingest = items.start_ingestion()?; + + for (k, v) in [10u8, 11, 12] + .into_iter() + .map(|i| (UserKey::new(&i.to_be_bytes()), UserValue::empty())) + { + ingest.write(k, v)?; + } + + ingest.finish()?; + } + assert_eq!(13, items.len()?); + assert_eq!(3, items.table_count()); + + { + let mut ingest = items.start_ingestion()?; + + for (k, v) in [13u8, 14] + .into_iter() + .map(|i| (UserKey::new(&i.to_be_bytes()), UserValue::empty())) + { + ingest.write(k, v)?; + } + + ingest.finish()?; + } + assert_eq!(15, items.len()?); + assert_eq!(4, items.table_count()); + + while !db.worker_pool.sender.is_empty() {} + assert_eq!(1, items.table_count()); + + Ok(()) +} diff --git a/crates/fjall/src/keyspace/write_delay.rs b/crates/fjall/src/keyspace/write_delay.rs new file mode 100644 index 000000000..1d96db09e --- /dev/null +++ b/crates/fjall/src/keyspace/write_delay.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +const STEP_SIZE: usize = 10_000; +const THRESHOLD: usize = 20; + +pub fn perform_write_stall(l0_runs: usize) { + if let THRESHOLD..30 = l0_runs { + let d = l0_runs - THRESHOLD; + + for _ in 0..(d * STEP_SIZE) { + std::hint::black_box(()); + } + } +} diff --git a/crates/fjall/src/lib.rs b/crates/fjall/src/lib.rs new file mode 100644 index 000000000..c241516c4 --- /dev/null +++ b/crates/fjall/src/lib.rs @@ -0,0 +1,180 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +//! Fjall is a log-structured embeddable key-value storage engine written in Rust. It features: +//! +//! - A thread-safe BTreeMap-like API +//! - 100% safe & stable Rust +//! - LSM-tree-based storage similar to `RocksDB` +//! - Range & prefix searching with forward and reverse iteration +//! - Keyspaces (a.k.a. column families) with cross-keyspace atomic semantics +//! - Built-in compression (default = `LZ4`) +//! - Serializable transactions (optional) +//! - Key-value separation for large blob use cases (optional) +//! - Automatic background maintenance +//! +//! It is not: +//! +//! - A standalone database server +//! - A relational or wide-column database: it has no notion of columns or query language +//! +//! Keys are limited to 65536 bytes, values are limited to 2^32 bytes. As is normal with any kind of storage engine, larger keys and values have a bigger performance impact. +//! +//! For the underlying LSM-tree implementation, see: . +//! +//! ## Basic usage +//! +//! ``` +//! use fjall::{PersistMode, Database, KeyspaceCreateOptions}; +//! # +//! # let folder = tempfile::tempdir().unwrap(); +//! +//! // A database may contain multiple keyspaces +//! // You should probably only use a single database for your application +//! let db = Database::builder(&folder).open()?; +//! // TxDatabase::builder for transactional semantics +//! +//! // Each keyspace is its own physical LSM-tree +//! let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?; +//! +//! // Write some data +//! items.insert("a", "hello")?; +//! +//! // And retrieve it +//! let bytes = items.get("a")?; +//! +//! // Or remove it again +//! items.remove("a")?; +//! +//! // Search by prefix +//! for kv in items.prefix("user1") { +//! // ... +//! } +//! +//! // Iterators implement DoubleEndedIterator, so you can search backwards, too! +//! for kv in items.prefix("prefix").rev() { +//! // ... +//! } +//! +//! // Search by range +//! for kv in items.range("a"..="z") { +//! // ... +//! } +//! +//! // Sync the journal to disk to make sure data is definitely durable +//! // When the database is dropped, it will try to persist with `PersistMode::SyncAll` as well +//! db.persist(PersistMode::SyncAll)?; +//! # +//! # Ok::<_, fjall::Error>(()) +//! ``` + +#![doc(html_logo_url = "https://raw.githubusercontent.com/fjall-rs/fjall/main/logo.png")] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/fjall-rs/fjall/main/logo.png")] +#![deny(unsafe_code)] +#![deny(clippy::all, missing_docs, clippy::cargo)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::indexing_slicing)] +#![warn(clippy::pedantic, clippy::nursery)] +#![warn(clippy::expect_used)] +#![allow(clippy::missing_const_for_fn, clippy::significant_drop_tightening)] +#![warn(clippy::multiple_crate_versions)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +macro_rules! fail_iter { + ($e:expr) => { + match $e { + Ok(v) => v, + Err(e) => return Some(Err(e.into())), + } + }; +} + +mod batch; +mod builder; + +/// Contains compaction strategies +pub mod compaction; + +mod db_config; + +#[cfg(feature = "__internal_whitebox")] +#[doc(hidden)] +pub mod drop; + +mod db; + +#[cfg(test)] +mod db_test; + +mod error; +mod file; +mod flush; +mod guard; +mod ingestion; +mod iter; +mod journal; +mod keyspace; +mod locked_file; +mod meta_keyspace; +mod path; +mod poison; +mod readable; +mod recovery; +mod snapshot; +mod snapshot_nonce; +mod snapshot_tracker; +mod stats; +mod supervisor; +mod tx; +mod version; +mod worker_pool; +mod write_buffer_manager; + +pub(crate) type HashMap = + std::collections::HashMap; + +/// Configuration policies +pub mod config { + pub use lsm_tree::config::{ + BlockSizePolicy, BloomConstructionPolicy, CompressionPolicy, FilterPolicy, + FilterPolicyEntry, HashRatioPolicy, PartitioningPolicy, PinningPolicy, + RestartIntervalPolicy, + }; +} + +pub use { + batch::WriteBatch as OwnedWriteBatch, + builder::Builder as DatabaseBuilder, + db::Database, + db_config::Config, + error::{Error, Result}, + guard::Guard, + iter::Iter, + journal::{error::RecoveryError as JournalRecoveryError, writer::PersistMode}, + keyspace::{options::CreateOptions as KeyspaceCreateOptions, Keyspace}, + readable::Readable, + snapshot::Snapshot, + version::FormatVersion, +}; + +pub use tx::single_writer::{ + SingleWriterTxKeyspace, TxDatabase as SingleWriterTxDatabase, + WriteTransaction as SingleWriterWriteTx, +}; + +pub use tx::optimistic::{ + Conflict, OptimisticTxDatabase, OptimisticTxKeyspace, WriteTransaction as OptimisticWriteTx, +}; + +#[doc(hidden)] +pub use lsm_tree::{AbstractTree, AnyTree, Error as LsmError, TreeType}; + +pub use lsm_tree::{ + CompressionType, KvPair, KvSeparationOptions, SeqNo, Slice, UserKey, UserValue, +}; + +/// Utility functions +pub mod util { + pub use lsm_tree::util::{prefix_to_range, prefixed_range}; +} diff --git a/crates/fjall/src/locked_file.rs b/crates/fjall/src/locked_file.rs new file mode 100644 index 000000000..34bcafafe --- /dev/null +++ b/crates/fjall/src/locked_file.rs @@ -0,0 +1,112 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::{ + fs::{File, OpenOptions}, + path::Path, + sync::Arc, +}; + +struct LockedFileGuardInner(File); + +impl Drop for LockedFileGuardInner { + fn drop(&mut self) { + log::debug!("Unlocking database lock"); + + self.0 + .unlock() + .inspect_err(|e| { + log::warn!("Failed to unlock database lock: {e:?}"); + }) + .ok(); + } +} + +#[derive(Clone)] +#[expect(unused)] +pub struct LockedFileGuard(Arc); + +impl LockedFileGuard { + pub fn create_new(path: &Path) -> crate::Result { + log::debug!("Acquiring database lock at {}", path.display()); + + let file = match File::create_new(path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + OpenOptions::new().read(true).write(true).open(path)? + } + e => e?, + }; + + file.try_lock().map_err(|e| match e { + std::fs::TryLockError::Error(e) => { + log::error!("Failed to acquire database lock - if this is expected, you can try opening again (maybe wait a little)"); + crate::Error::Io(e) + } + std::fs::TryLockError::WouldBlock => crate::Error::Locked, + })?; + + Ok(Self(Arc::new(LockedFileGuardInner(file)))) + } + + pub fn try_acquire(path: &Path) -> crate::Result { + const RETRIES: usize = 3; + + log::debug!("Acquiring database lock at {}", path.display()); + + let file = OpenOptions::new().read(true).write(true).open(path)?; + + for i in 1..=RETRIES { + if let Err(e) = file.try_lock() { + match e { + std::fs::TryLockError::Error(e) => { + log::error!("Failed to acquire database lock - if this is expected, you can try opening again (maybe wait a little)"); + return Err(crate::Error::Io(e)); + } + std::fs::TryLockError::WouldBlock => { + if i == RETRIES { + return Err(crate::Error::Locked); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + } + } else { + // Success + break; + } + } + + Ok(Self(Arc::new(LockedFileGuardInner(file)))) + } +} + +#[cfg(test)] +mod tests { + use super::LockedFileGuard; + use std::fs::File; + + #[test] + fn create_new_acquires_lock_when_file_already_exists() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("lock"); + + File::create(&path)?; + + let _guard = LockedFileGuard::create_new(&path)?; + + Ok(()) + } + + #[test] + fn try_acquire_acquires_lock_on_existing_file() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("lock"); + + File::create(&path)?; + + let _guard = LockedFileGuard::try_acquire(&path)?; + + Ok(()) + } +} diff --git a/crates/fjall/src/meta_keyspace.rs b/crates/fjall/src/meta_keyspace.rs new file mode 100644 index 000000000..0a3e1cb2e --- /dev/null +++ b/crates/fjall/src/meta_keyspace.rs @@ -0,0 +1,483 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{db::Keyspaces, keyspace::InternalKeyspaceId, Keyspace}; +use byteview::StrView; +use lsm_tree::{AbstractTree, AnyTree, SeqNo, SequenceNumberCounter, UserValue}; +use std::sync::{Arc, RwLock, RwLockWriteGuard}; + +pub fn encode_config_key(keyspace_id: InternalKeyspaceId, name: &str) -> crate::UserKey { + use byteorder::{WriteBytesExt, BE}; + use std::io::Write; + + #[expect(unsafe_code)] + let mut key = unsafe { + crate::UserKey::builder_unzeroed(1 + std::mem::size_of::() + name.len()) + }; + + #[expect( + clippy::unwrap_used, + reason = "writing into preallocated buffer should not trigger IO or EOF error" + )] + { + let mut writer = &mut key[..]; + writer.write_u8(b'c').unwrap(); + writer.write_u64::(keyspace_id).unwrap(); + writer.write_all(name.as_bytes()).unwrap(); + } + + key.freeze().into() +} + +/// The meta keyspace keeps mappings of keyspace names to their internal IDs and configurations +/// +/// The meta keyspace is always keyspace #0. +/// +/// The schema looks roughtly like this: +/// +/// ```md +/// 'c' + + -> +/// 'n' + -> +/// ``` +#[derive(Clone)] +pub struct MetaKeyspace { + /// The actual backing LSM-tree + /// + /// We never commit data through the WAL but write tables directly using + /// lsm-tree's ingestion API. + inner: AnyTree, + + /// Dictionary of all keyspaces + #[doc(hidden)] + pub keyspaces: Arc>, + + seqno_generator: SequenceNumberCounter, + visible_seqno: SequenceNumberCounter, +} + +impl MetaKeyspace { + pub(crate) fn new( + inner: AnyTree, + keyspaces: Arc>, + seqno_generator: SequenceNumberCounter, + visible_seqno: SequenceNumberCounter, + ) -> Self { + Self { + inner, + keyspaces, + seqno_generator, + visible_seqno, + } + } + + #[cfg(test)] + fn len(&self) -> crate::Result { + self.inner.len(SeqNo::MAX, None).map_err(Into::into) + } + + pub(crate) fn get_kv_for_config( + &self, + keyspace_id: InternalKeyspaceId, + name: &str, + ) -> crate::Result> { + let key = encode_config_key(keyspace_id, name); + self.inner.get(key, SeqNo::MAX).map_err(Into::into) + } + + fn maintenance(&self) -> crate::Result<()> { + self.inner + .compact( + Arc::new( + crate::compaction::Leveled::default() + .with_l0_threshold(2) + .with_level_ratio_policy(vec![20.0]), + ), + 0, + ) + .map_err(Into::into) + } + + pub(crate) fn create_keyspace( + &self, + keyspace_id: InternalKeyspaceId, + name: &str, + keyspace: Keyspace, + mut keyspaces: RwLockWriteGuard<'_, Keyspaces>, + ) -> crate::Result<()> { + let mut kvs = keyspace.config.encode_kvs(keyspace_id); + + kvs.push({ + let mut key: Vec = + Vec::with_capacity(std::mem::size_of::() + 1); + key.push(b'n'); + key.extend(keyspace_id.to_be_bytes()); + + let value = UserValue::new(name.as_bytes()); + + (key.into(), value) + }); + + kvs.sort_by(|(a, _), (b, _)| a.cmp(b)); + + #[cfg(debug_assertions)] + { + assert!( + kvs.is_sorted_by_key(|(k, _)| k), + "config KVs need to be sorted by key", + ); + } + + let mut ingestion = self.inner.ingestion()?; + + for kv in kvs { + ingestion.write(kv.0, kv.1)?; + } + + ingestion.finish()?; + + keyspaces.insert(name.into(), keyspace); + + self.maintenance() + .inspect_err(|e| { + log::warn!("Meta keyspace maintenance failed: {e:?}"); + }) + .ok(); + + Ok(()) + } + + pub(crate) fn remove_keyspace(&self, name: &str) -> crate::Result<()> { + let mut lock = self.keyspaces.write().expect("lock is poisoned"); + + let Some(keyspace) = lock.get(name) else { + return Ok(()); + }; + + let seqno = self.seqno_generator.next(); + + let mut ingestion = self.inner.ingestion()?; + { + // Remove all config KVs + let pfx: Vec = { + let mut v = vec![]; + v.push(b'c'); + v.extend(keyspace.id.to_be_bytes()); + v + }; + + for config_kv in self.inner.prefix(pfx, SeqNo::MAX, None) { + use lsm_tree::Guard; + + let key = config_kv.key()?; + ingestion.write_tombstone(key)?; + } + + // Remove ID -> name mapping + let mut key: Vec = + Vec::with_capacity(std::mem::size_of::() + 1); + key.push(b'n'); + key.extend(keyspace.id.to_be_bytes()); + ingestion.write_tombstone(key)?; + } + ingestion.finish()?; + + self.visible_seqno.fetch_max(seqno + 1); + + lock.remove(name); + + self.maintenance() + .inspect_err(|e| { + log::warn!("Meta keyspace maintenance failed: {e:?}"); + }) + .ok(); + + Ok(()) + } + + pub(crate) fn resolve_id(&self, id: InternalKeyspaceId) -> crate::Result> { + #[expect(unsafe_code, clippy::indexing_slicing)] + let key = { + let mut builder = unsafe { + lsm_tree::Slice::builder_unzeroed(std::mem::size_of::() + 1) + }; + builder[0] = b'n'; + builder[1..].copy_from_slice(&id.to_be_bytes()); + builder.freeze() + }; + + Ok(self.inner.get(key, SeqNo::MAX)?.map(|v| { + #[expect(clippy::expect_used, reason = "keyspace open only accepts &str")] + let name = std::str::from_utf8(&v).expect("should be utf-8"); + + name.into() + })) + } + + pub(crate) fn keyspace_exists(&self, name: &str) -> bool { + self.keyspaces + .read() + .expect("lock is poisoned") + .contains_key(name) + } +} + +#[cfg(test)] +mod tests { + use crate::{Database, Guard, KeyspaceCreateOptions, Readable, SingleWriterTxDatabase}; + use test_log::test; + + const ITEM_COUNT: usize = 10; + + #[test] + fn meta_keyspace_smoke_test() -> crate::Result<()> { + use crate::AbstractTree; + use lsm_tree::SeqNo; + + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + assert_eq!( + None, + db.meta_keyspace + .inner + .get(&['n' as u8, 0, 0, 0, 0, 0, 0, 0, 1], SeqNo::MAX)? + .as_deref(), + ); + + let _ks = db.keyspace("items", Default::default)?; + + assert_eq!( + Some(b"items".as_slice()), + db.meta_keyspace + .inner + .get(&['n' as u8, 0, 0, 0, 0, 0, 0, 0, 1], SeqNo::MAX)? + .as_deref(), + ); + assert_eq!( + Some([3u8].as_slice()), + db.meta_keyspace + .inner + .get( + &[ + 'c' as u8, 0, 0, 0, 0, 0, 0, 0, 1, 'v' as u8, 'e' as u8, 'r' as u8, + 's' as u8, 'i' as u8, 'o' as u8, 'n' as u8 + ], + SeqNo::MAX + )? + .as_deref(), + ); + } + + // Config should still exists after recovery, even without opening the keyspace + { + let db = Database::builder(&folder).open()?; + + assert_eq!( + Some(b"items".as_slice()), + db.meta_keyspace + .inner + .get(&['n' as u8, 0, 0, 0, 0, 0, 0, 0, 1], SeqNo::MAX)? + .as_deref(), + ); + assert_eq!( + Some([3u8].as_slice()), + db.meta_keyspace + .inner + .get( + &[ + 'c' as u8, 0, 0, 0, 0, 0, 0, 0, 1, 'v' as u8, 'e' as u8, 'r' as u8, + 's' as u8, 'i' as u8, 'o' as u8, 'n' as u8 + ], + SeqNo::MAX + )? + .as_deref(), + ); + } + + Ok(()) + } + + #[test] + fn keyspace_delete() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let path; + + { + let db = Database::builder(&folder).open()?; + + assert_eq!(0, db.meta_keyspace.len()?); + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + path = tree.path().to_path_buf(); + + assert!(path.try_exists()?); + assert!(db.meta_keyspace.len()? > 0); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(Guard::key).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(Guard::key).count(), + ITEM_COUNT * 2, + ); + } + + for _ in 0..3 { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert!(path.try_exists()?); + assert!(db.meta_keyspace.len()? > 0); + + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(Guard::key).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(Guard::key).count(), + ITEM_COUNT * 2, + ); + } + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert!(path.try_exists()?); + assert!(db.meta_keyspace.len()? > 0); + + db.delete_keyspace(tree)?; + assert!(!path.try_exists()?); + assert_eq!(0, db.meta_keyspace.len()?); + } + + { + let db = Database::builder(&folder).open()?; + assert!(!path.try_exists()?); + assert_eq!(0, db.meta_keyspace.len()?); + } + + Ok(()) + } + + #[test] + fn tx_keyspace_delete() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let path; + + { + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + path = tree.path(); + + assert!(path.try_exists()?); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + assert_eq!(db.read_tx().len(&tree)?, ITEM_COUNT * 2); + assert_eq!( + db.read_tx().iter(&tree).flat_map(Guard::key).count(), + ITEM_COUNT * 2, + ); + assert_eq!( + db.read_tx().iter(&tree).rev().flat_map(Guard::key).count(), + ITEM_COUNT * 2, + ); + } + + for _ in 0..3 { + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert_eq!(db.read_tx().len(&tree)?, ITEM_COUNT * 2); + assert_eq!( + db.read_tx().iter(&tree).flat_map(Guard::key).count(), + ITEM_COUNT * 2, + ); + assert_eq!( + db.read_tx().iter(&tree).rev().flat_map(Guard::key).count(), + ITEM_COUNT * 2, + ); + + assert!(path.try_exists()?); + } + + { + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + { + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert!(path.try_exists()?); + + db.inner().delete_keyspace(tree.inner().clone())?; + } + + assert!(!path.try_exists()?); + } + + { + let _db = Database::builder(&folder).open()?; + assert!(!path.try_exists()?); + } + + Ok(()) + } + + #[test] + fn keyspace_delete_and_reopening_behavior() -> crate::Result<()> { + let keyspace_name = "default"; + let folder = tempfile::tempdir()?; + + let keyspace_exists = |id: u64| -> std::io::Result { + folder + .path() + .join("keyspaces") + .join(id.to_string()) + .try_exists() + }; + + let db = Database::builder(&folder).open()?; + assert!(!keyspace_exists(1)?); + + let keyspace = db.keyspace(keyspace_name, KeyspaceCreateOptions::default)?; + assert!(keyspace_exists(1)?); + + db.delete_keyspace(keyspace)?; + assert!(!keyspace_exists(1)?); + + assert!(db + .keyspace("default", KeyspaceCreateOptions::default) + .is_ok()); + assert!(keyspace_exists(2)?); + + Ok(()) + } +} diff --git a/crates/fjall/src/path.rs b/crates/fjall/src/path.rs new file mode 100644 index 000000000..fb2af6d21 --- /dev/null +++ b/crates/fjall/src/path.rs @@ -0,0 +1,10 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::path::{Path, PathBuf}; + +pub fn absolute_path(path: &Path) -> PathBuf { + #[expect(clippy::expect_used, reason = "nothing we can do")] + std::path::absolute(path).expect("should be absolute path") +} diff --git a/crates/fjall/src/poison.rs b/crates/fjall/src/poison.rs new file mode 100644 index 000000000..5744b6c63 --- /dev/null +++ b/crates/fjall/src/poison.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::sync::{atomic::AtomicBool, Arc}; + +#[derive(Clone, Debug, Default)] +pub struct PoisonSignal(Arc); + +impl PoisonSignal { + pub fn is_poisoned(&self) -> bool { + self.0.load(std::sync::atomic::Ordering::Acquire) + } + + pub fn poison(&self) { + self.0.store(true, std::sync::atomic::Ordering::Release); + } +} + +/// RAII guard to catch panics in background workers and poison a database +#[derive(Clone)] +pub struct PoisonDart(PoisonSignal); + +impl PoisonDart { + pub fn new(signal: PoisonSignal) -> Self { + Self(signal) + } + + pub fn poison(&self) { + self.0.poison(); + } +} + +impl Drop for PoisonDart { + fn drop(&mut self) { + if std::thread::panicking() { + log::error!("Poisoning database because of panic in background worker"); + self.poison(); + } + } +} diff --git a/crates/fjall/src/readable.rs b/crates/fjall/src/readable.rs new file mode 100644 index 000000000..1680f78a6 --- /dev/null +++ b/crates/fjall/src/readable.rs @@ -0,0 +1,301 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{Guard, Iter, Keyspace}; +use lsm_tree::UserValue; +use std::ops::RangeBounds; + +/// Readable snapshot +/// +/// Can be used to pass a write transaction into a function that expects a snapshot. +pub trait Readable { + /// Retrieves an item from the snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let snapshot = db.snapshot(); + /// let item = snapshot.get(&tree, "a")?; + /// assert_eq!(Some("my_value".as_bytes().into()), item); + /// + /// tree.insert("b", "my_updated_value")?; + /// + /// // Repeatable read + /// let item = snapshot.get(&tree, "a")?; + /// assert_eq!(Some("my_value".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn get>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result>; + + /// Returns `true` if the snapshot contains the specified key. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let snapshot = db.snapshot(); + /// assert!(snapshot.contains_key(&tree, "a")?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn contains_key>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result; + + /// Returns the first key-value pair in the snapshot. + /// The key in this pair is the minimum key in the snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("1", "abc")?; + /// tree.insert("3", "abc")?; + /// tree.insert("5", "abc")?; + /// + /// let key = db.snapshot().first_key_value(&tree).expect("item should exist").key()?; + /// assert_eq!(&*key, "1".as_bytes()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + fn first_key_value(&self, keyspace: impl AsRef) -> Option; + + /// Returns the last key-value pair in the snapshot. + /// The key in this pair is the maximum key in the snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("1", "abc")?; + /// tree.insert("3", "abc")?; + /// tree.insert("5", "abc")?; + /// + /// let key = db.snapshot().last_key_value(&tree).expect("item should exist").key()?; + /// assert_eq!(&*key, "5".as_bytes()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + fn last_key_value(&self, keyspace: impl AsRef) -> Option; + + /// Retrieves the size of an item from the snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let snapshot = db.snapshot(); + /// let item = snapshot.size_of(&tree, "a")?.unwrap_or_default(); + /// assert_eq!("my_value".len() as u32, item); + /// + /// tree.insert("b", "my_updated_value")?; + /// + /// // Repeatable read + /// let item = snapshot.size_of(&tree, "a")?.unwrap_or_default(); + /// assert_eq!("my_value".len() as u32, item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn size_of>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result>; + + /// Returns `true` if the parkeyspacetition is empty. + /// + /// This operation has O(log N) complexity. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert!(db.snapshot().is_empty(&tree)?); + /// + /// tree.insert("a", "abc")?; + /// assert!(!db.snapshot().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn is_empty(&self, keyspace: impl AsRef) -> crate::Result { + Ok(self + .first_key_value(keyspace) + .map(Guard::key) + .transpose()? + .is_none()) + } + + /// Iterates over the snapshot. + /// + /// Avoid using this function, or limit it as otherwise it may scan a lot of items. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// tree.insert("f", "abc")?; + /// tree.insert("g", "abc")?; + /// + /// assert_eq!(3, db.snapshot().iter(&tree).count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + fn iter(&self, keyspace: impl AsRef) -> Iter; + + /// Scans the entire keyspace, returning the number of items. + /// + /// # Caution + /// + /// This operation scans the entire keyspace: O(n) complexity! + /// + /// Never, under any circumstances, use .`len()` == 0 to check + /// if the keyspace is empty, use [`Readable::is_empty`] instead. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// tree.insert("b", "my_value2")?; + /// + /// let snapshot = db.snapshot(); + /// assert_eq!(2, snapshot.len(&tree)?); + /// + /// tree.insert("c", "my_value3")?; + /// + /// // Repeatable read + /// assert_eq!(2, snapshot.len(&tree)?); + /// + /// // Start new snapshot + /// let snapshot = db.snapshot(); + /// assert_eq!(3, snapshot.len(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn len(&self, keyspace: impl AsRef) -> crate::Result { + let mut count = 0; + + for guard in self.iter(keyspace) { + let _ = guard.key()?; + count += 1; + } + + Ok(count) + } + + /// Iterates over a range of the snapshot. + /// + /// Avoid using full or unbounded ranges as they may scan a lot of items (unless limited). + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// tree.insert("f", "abc")?; + /// tree.insert("g", "abc")?; + /// + /// assert_eq!(2, db.snapshot().range(&tree, "a"..="f").count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + fn range, R: RangeBounds>( + &self, + keyspace: impl AsRef, + range: R, + ) -> Iter; + + /// Iterates over a prefixed set of the snapshot. + /// + /// Avoid using an empty prefix as it may scan a lot of items (unless limited). + /// + /// # Examples + /// + /// ``` + /// # use fjall::{Database, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = Database::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// tree.insert("ab", "abc")?; + /// tree.insert("abc", "abc")?; + /// + /// assert_eq!(2, db.snapshot().prefix(&tree, "ab").count()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + fn prefix>(&self, keyspace: impl AsRef, prefix: K) -> Iter; +} diff --git a/crates/fjall/src/recovery.rs b/crates/fjall/src/recovery.rs new file mode 100644 index 000000000..629f87661 --- /dev/null +++ b/crates/fjall/src/recovery.rs @@ -0,0 +1,270 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + file::{KEYSPACES_FOLDER, LSM_CURRENT_VERSION_MARKER}, + journal::{ + batch_reader::JournalBatchReader, manager::EvictionWatermark, reader::JournalReader, + }, + keyspace::{ + apply_to_base_config, options::CreateOptions as KeyspaceCreateOptions, InternalKeyspaceId, + }, + meta_keyspace::MetaKeyspace, + Database, HashMap, Keyspace, +}; +use lsm_tree::AbstractTree; +use std::path::PathBuf; + +/// Recovers keyspaces +pub fn recover_keyspaces(db: &Database, meta_keyspace: &MetaKeyspace) -> crate::Result<()> { + let keyspaces_folder = db.config.path.join(KEYSPACES_FOLDER); + + log::trace!("Recovering keyspaces in {}", keyspaces_folder.display()); + + #[expect(clippy::expect_used)] + let mut keyspaces_lock = db.supervisor.keyspaces.write().expect("lock is poisoned"); + + let mut highest_id = 1; + + for dirent in std::fs::read_dir(&keyspaces_folder)? { + let dirent = dirent?; + let keyspace_path = dirent.path(); + + if dirent.file_type()?.is_file() { + log::warn!( + "Found stray file {} in keyspaces folder", + keyspace_path.display(), + ); + continue; + } + + let keyspace_id = dirent + .file_name() + .to_str() + .expect("should be valid keyspace name") + .parse::() + .expect("should be valid integer"); + + // NOTE: Is meta keyspace + if keyspace_id == 0 { + continue; + } + + highest_id = highest_id.max(keyspace_id); + + let Some(keyspace_name) = meta_keyspace.resolve_id(keyspace_id)? else { + log::debug!("Deleting unreferenced keyspace id={keyspace_id}"); + std::fs::remove_dir_all(keyspace_path)?; + continue; + }; + + log::trace!("Recovering keyspace {keyspace_id}"); + + // NOTE: Check for marker, maybe the keyspace is not fully initialized + if !keyspace_path + .join(LSM_CURRENT_VERSION_MARKER) + .try_exists()? + { + log::debug!("Deleting uninitialized keyspace {keyspace_name:?}"); + std::fs::remove_dir_all(keyspace_path)?; + continue; + } + + let path = keyspaces_folder.join(keyspace_id.to_string()); + + let mut recovered_config = KeyspaceCreateOptions::from_kvs(keyspace_id, &db.meta_keyspace)?; + + // Install compaction filter factory if needed + if let Some(f) = db + .config + .compaction_filter_factory_assigner + .as_ref() + .and_then(|f| f(&keyspace_name)) + { + recovered_config = recovered_config.with_compaction_filter_factory(f); + } + + let base_config = lsm_tree::Config::new( + path, + db.supervisor.seqno.clone(), + db.supervisor.snapshot_tracker.get_ref(), + ) + .use_descriptor_table(db.config.descriptor_table.clone()) + .use_cache(db.config.cache.clone()); + + let base_config = apply_to_base_config(base_config, &recovered_config); + + let tree = base_config.open()?; + + let keyspace = Keyspace::from_database( + keyspace_id, + db, + tree, + keyspace_name.clone(), + recovered_config, + ); + + // Add keyspace to dictionary + keyspaces_lock.insert(keyspace_name.clone(), keyspace.clone()); + + log::trace!("Recovered keyspace {keyspace_name:?}"); + } + + db.keyspace_id_counter.set(highest_id + 1); + + Ok(()) +} + +#[expect(clippy::too_many_lines)] +pub fn recover_sealed_memtables( + db: &Database, + sealed_journal_paths: &[PathBuf], +) -> crate::Result<()> { + #[expect(clippy::expect_used)] + let mut journal_manager_lock = db + .supervisor + .journal_manager + .write() + .expect("lock is poisoned"); + + #[expect(clippy::expect_used)] + let keyspaces_lock = db.supervisor.keyspaces.read().expect("lock is poisoned"); + + for journal_path in sealed_journal_paths { + log::debug!("Recovering sealed journal: {}", journal_path.display()); + + let journal_size = journal_path.metadata()?.len(); + + log::debug!("Reading sealed journal at {}", journal_path.display()); + + let raw_reader = JournalReader::new(journal_path)?; + let reader = JournalBatchReader::new(raw_reader); + + let mut watermarks: HashMap<_, EvictionWatermark> = HashMap::default(); + + for batch in reader { + let batch = batch?; + + for item in batch.items { + let Some(keyspace_name) = db.meta_keyspace.resolve_id(item.keyspace_id)? else { + continue; + }; + + let Some(handle) = keyspaces_lock.get(&keyspace_name) else { + continue; + }; + + let tree = &handle.tree; + + watermarks + .entry(item.keyspace_id) + .and_modify(|prev| { + prev.lsn = prev.lsn.max(batch.seqno); + }) + .or_insert_with(|| EvictionWatermark { + keyspace: handle.clone(), + lsn: batch.seqno, + }); + + match item.value_type { + lsm_tree::ValueType::Value => { + tree.insert(item.key, item.value, batch.seqno); + } + lsm_tree::ValueType::Tombstone => { + tree.remove(item.key, batch.seqno); + } + lsm_tree::ValueType::WeakTombstone => { + tree.remove_weak(item.key, batch.seqno); + } + lsm_tree::ValueType::Indirection => { + unreachable!() + } + } + } + + for keyspace_id in &batch.cleared_keyspaces { + let Some(keyspace_name) = db.meta_keyspace.resolve_id(*keyspace_id)? else { + continue; + }; + + let Some(handle) = keyspaces_lock.get(&keyspace_name) else { + continue; + }; + + watermarks + .entry(*keyspace_id) + .and_modify(|prev| { + prev.lsn = prev.lsn.max(batch.seqno); + }) + .or_insert_with(|| EvictionWatermark { + keyspace: handle.clone(), + lsn: batch.seqno, + }); + + handle.tree.clear().inspect_err(|e| { + log::error!("Keyspace clear failed during recovery: {e}"); + })?; + } + } + + log::debug!("Sealing recovered memtables"); + let mut recovered_count = 0; + + for wm in watermarks.values() { + let tree = &wm.keyspace.tree; + + let keyspace_lsn = tree.get_highest_persisted_seqno(); + + // IMPORTANT: Apply the sealed memtable only if it contains data newer than the + // keyspace's persisted LSN. + let should_skip_sealed_memtable = + keyspace_lsn.is_some_and(|keyspace_lsn| keyspace_lsn >= wm.lsn); + + if should_skip_sealed_memtable { + log::trace!( + "Keyspace {:?} has higher seqno ({keyspace_lsn:?}), skipping", + wm.keyspace.name, + ); + + tree.clear_active_memtable(); + } else if let Some(sealed_memtable) = tree.rotate_memtable() { + log::trace!("Sealed active memtable of keyspace {:?}", wm.keyspace.name); + + assert_eq!( + Some(wm.lsn), + sealed_memtable.get_highest_seqno(), + "memtable lsn does not match what was recovered - this is a bug", + ); + + // Maybe the memtable has a higher seqno, so try to set to maximum + let maybe_next_seqno = tree.get_highest_seqno().map(|x| x + 1).unwrap_or_default(); + db.supervisor.seqno.fetch_max(maybe_next_seqno); + + log::debug!("Database seqno is now {}", db.supervisor.seqno.get()); + + // IMPORTANT: Add sealed memtable size to current write buffer size + db.supervisor + .write_buffer_size + .allocate(sealed_memtable.size()); + + // TODO: unit test write buffer size after recovery + + recovered_count += 1; + } + } + + log::debug!("Recovered {recovered_count} sealed memtables"); + + // IMPORTANT: Add sealed journal to journal manager + journal_manager_lock.enqueue(crate::journal::manager::Item { + watermarks: watermarks.into_values().collect(), + path: journal_path.clone(), + size_in_bytes: journal_size, + }); + + log::debug!("Requeued sealed journal at {}", journal_path.display()); + } + + Ok(()) +} diff --git a/crates/fjall/src/snapshot.rs b/crates/fjall/src/snapshot.rs new file mode 100644 index 000000000..920bb328d --- /dev/null +++ b/crates/fjall/src/snapshot.rs @@ -0,0 +1,105 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{snapshot_nonce::SnapshotNonce, Guard, Iter, Keyspace, Readable}; +use lsm_tree::{AbstractTree, SeqNo, UserValue}; +use std::ops::RangeBounds; + +/// A cross-keyspace snapshot +/// +/// Snapshots keep a consistent view of the database at the time, +/// meaning old data will not be dropped until it is not referenced by any active snapshot. +/// For that reason, you should try to keep snapshots short-lived, and make sure they +/// are not held somewhere *forever*. +#[clippy::has_significant_drop] +#[derive(Clone)] +pub struct Snapshot { + pub(crate) nonce: SnapshotNonce, +} + +impl Snapshot { + pub(crate) fn new(nonce: SnapshotNonce) -> Self { + Self { nonce } + } + + #[doc(hidden)] + #[must_use] + pub fn seqno(&self) -> SeqNo { + self.nonce.instant + } +} + +impl Readable for Snapshot { + fn get>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + keyspace + .as_ref() + .tree + .get(key, self.nonce.instant) + .map_err(Into::into) + } + + fn contains_key>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result { + keyspace + .as_ref() + .tree + .contains_key(key, self.nonce.instant) + .map_err(Into::into) + } + + fn first_key_value(&self, keyspace: impl AsRef) -> Option { + self.iter(keyspace).next() + } + + fn last_key_value(&self, keyspace: impl AsRef) -> Option { + self.iter(keyspace).next_back() + } + + fn size_of>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + keyspace + .as_ref() + .tree + .size_of(key, self.nonce.instant) + .map_err(Into::into) + } + + fn iter(&self, keyspace: impl AsRef) -> Iter { + let iter = keyspace.as_ref().tree.iter(self.nonce.instant, None); + + Iter::new(self.nonce.clone(), iter) + } + + fn range, R: RangeBounds>( + &self, + keyspace: impl AsRef, + range: R, + ) -> Iter { + let iter = keyspace + .as_ref() + .tree + .range(range, self.nonce.instant, None); + + Iter::new(self.nonce.clone(), iter) + } + + fn prefix>(&self, keyspace: impl AsRef, prefix: K) -> Iter { + let iter = keyspace + .as_ref() + .tree + .prefix(prefix, self.nonce.instant, None); + + Iter::new(self.nonce.clone(), iter) + } +} diff --git a/crates/fjall/src/snapshot_nonce.rs b/crates/fjall/src/snapshot_nonce.rs new file mode 100644 index 000000000..b8e47ad0f --- /dev/null +++ b/crates/fjall/src/snapshot_nonce.rs @@ -0,0 +1,38 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{snapshot_tracker::SnapshotTracker, SeqNo}; + +/// Holds a snapshot instant and automatically frees it from the snapshot tracker when dropped +pub struct SnapshotNonce { + pub(crate) instant: SeqNo, + tracker: SnapshotTracker, +} + +impl std::fmt::Debug for SnapshotNonce { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "SnapshotNonce({})", self.instant) + } +} + +impl SnapshotNonce { + pub(crate) fn new(seqno: SeqNo, tracker: SnapshotTracker) -> Self { + Self { + instant: seqno, + tracker, + } + } +} + +impl Clone for SnapshotNonce { + fn clone(&self) -> Self { + self.tracker.clone_snapshot(self) + } +} + +impl Drop for SnapshotNonce { + fn drop(&mut self) { + self.tracker.close(self); + } +} diff --git a/crates/fjall/src/snapshot_tracker.rs b/crates/fjall/src/snapshot_tracker.rs new file mode 100644 index 000000000..3d68fec2c --- /dev/null +++ b/crates/fjall/src/snapshot_tracker.rs @@ -0,0 +1,405 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{snapshot_nonce::SnapshotNonce, SeqNo}; +use dashmap::DashMap; +use lsm_tree::SequenceNumberCounter; +use std::sync::{atomic::AtomicU64, Arc, RwLock}; + +/// Keeps track of open snapshots +pub struct SnapshotTrackerInner { + seqno: SequenceNumberCounter, + + gc_lock: RwLock<()>, + + // TODO: maybe use rustc_hash or ahash + data: DashMap, + + freed_count: AtomicU64, + + pub(crate) lowest_freed_instant: AtomicU64, +} + +#[derive(Clone)] +pub struct SnapshotTracker(Arc); + +impl std::ops::Deref for SnapshotTracker { + type Target = SnapshotTrackerInner; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl SnapshotTracker { + pub fn new(seqno: SequenceNumberCounter) -> Self { + Self(Arc::new(SnapshotTrackerInner { + data: DashMap::default(), + freed_count: AtomicU64::default(), + lowest_freed_instant: AtomicU64::default(), + seqno, + gc_lock: RwLock::default(), + })) + } + + pub fn get_ref(&self) -> SequenceNumberCounter { + self.seqno.clone() + } + + /// Used in database recovery. + /// + /// # Caution + /// + /// Don't use this to get a snapshot read. + pub fn get(&self) -> SeqNo { + self.seqno.get() + } + + /// Used in database recovery. + pub fn set(&self, value: SeqNo) { + self.seqno.fetch_max(value); + } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn open_snapshots(&self) -> usize { + self.data.iter().map(|r| *r.value()).sum() + } + + pub fn open(&self) -> SnapshotNonce { + #[expect(clippy::expect_used)] + let _lock = self.gc_lock.read().expect("lock is poisoned"); + + let seqno = self.seqno.get(); + + self.data + .entry(seqno) + .and_modify(|x| { + *x += 1; + }) + .or_insert(1); + + SnapshotNonce::new(seqno, self.clone()) + } + + pub fn clone_snapshot(&self, nonce: &SnapshotNonce) -> SnapshotNonce { + #[expect(clippy::expect_used)] + let _lock = self.gc_lock.read().expect("lock is poisoned"); + + self.data + .entry(nonce.instant) + .and_modify(|x| { + *x += 1; + }) + .or_insert(1); + + SnapshotNonce::new(nonce.instant, self.clone()) + } + + pub fn close(&self, nonce: &SnapshotNonce) { + self.close_raw(nonce.instant); + } + + pub(crate) fn close_raw(&self, instant: SeqNo) { + #[expect(clippy::expect_used)] + let lock = self.gc_lock.read().expect("lock is poisoned"); + + self.data.alter(&instant, |_, v| v.saturating_sub(1)); + + let freed = self + .freed_count + .fetch_add(1, std::sync::atomic::Ordering::AcqRel) + + 1; + + drop(lock); + + if freed.is_multiple_of(10_000) { + self.gc(); + } + } + + /// Publish write completion + pub fn publish(&self, batch_seqno: SeqNo) { + self.seqno.fetch_max(batch_seqno + 1); + } + + // TODO: after recovery, we may need to set the GC watermark once to current_seqno - 1 + // so there cannot be compactions scheduled immediately with gc_watermark=0 + pub fn get_seqno_safe_to_gc(&self) -> SeqNo { + self.lowest_freed_instant + .load(std::sync::atomic::Ordering::Acquire) + } + + pub(crate) fn pullup(&self) { + #[expect(clippy::expect_used)] + let _lock = self.gc_lock.write().expect("lock is poisoned"); + + if self.data.is_empty() { + self.lowest_freed_instant.store( + self.seqno.get().saturating_sub(1), + std::sync::atomic::Ordering::Release, + ); + } + } + + pub(crate) fn gc(&self) { + #[expect(clippy::expect_used)] + let _lock = self.gc_lock.write().expect("lock is poisoned"); + + let seqno_threshold = self.seqno.get(); + + let mut lowest_retained = 0; + let mut none_retained = true; + + self.data.retain(|&k, v| { + let should_be_retained = *v > 0 || k >= seqno_threshold; + + if should_be_retained { + lowest_retained = match lowest_retained { + 0 => k, + lo => lo.min(k), + }; + none_retained = false; + } + + should_be_retained + }); + + if none_retained { + lowest_retained = seqno_threshold; + } + + self.lowest_freed_instant.fetch_max( + lowest_retained.saturating_sub(1), + std::sync::atomic::Ordering::AcqRel, + ); + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn snapshot_tracker_mvcc_watermark_1() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + + let db = crate::Database::builder(&dir).open()?; + let tree = db.keyspace("default", Default::default)?; + + tree.insert("a", "a")?; + tree.insert("a", "b")?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + + tree.rotate_memtable_and_wait()?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + + db.supervisor.snapshot_tracker.gc(); + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + + Ok(()) + } + + #[test] + fn snapshot_tracker_mvcc_watermark_2() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + + let db = crate::Database::builder(&dir).open()?; + let tree = db.keyspace("default", Default::default)?; + + tree.insert("a", "a")?; + tree.insert("a", "b")?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + + db.supervisor.snapshot_tracker.gc(); + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + + tree.rotate_memtable_and_wait()?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + + Ok(()) + } + + #[test] + fn snapshot_tracker_mvcc_watermark_3() -> crate::Result<()> { + use crate::Readable; + + let dir = tempfile::tempdir()?; + + let db = crate::Database::builder(&dir).open()?; + let tree = db.keyspace("default", Default::default)?; + + tree.insert("a", "a")?; + + let snapshot = db.snapshot(); + + tree.insert("a", "b")?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + assert_eq!(b"a", &*snapshot.get(&tree, "a")?.unwrap()); + + db.supervisor.snapshot_tracker.gc(); + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + assert_eq!(b"a", &*snapshot.get(&tree, "a")?.unwrap()); + + tree.rotate_memtable_and_wait()?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + assert_eq!(b"a", &*snapshot.get(&tree, "a")?.unwrap()); + + Ok(()) + } + + #[test] + fn snapshot_tracker_mvcc_watermark_4() -> crate::Result<()> { + use crate::Readable; + + let dir = tempfile::tempdir()?; + + let db = crate::Database::builder(&dir).open()?; + let tree = db.keyspace("default", Default::default)?; + + tree.insert("a", "a")?; + + let snapshot = db.snapshot(); + + tree.insert("a", "b")?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + assert_eq!(b"a", &*snapshot.get(&tree, "a")?.unwrap()); + + tree.rotate_memtable_and_wait()?; + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + assert_eq!(b"a", &*snapshot.get(&tree, "a")?.unwrap()); + + db.supervisor.snapshot_tracker.gc(); + assert_eq!(b"b", &*tree.get("a")?.unwrap()); + assert_eq!(b"a", &*snapshot.get(&tree, "a")?.unwrap()); + + Ok(()) + } + + #[test] + fn snapshot_tracker_basic() { + let global_seqno = SequenceNumberCounter::default(); + + let map = SnapshotTracker::new(global_seqno.clone()); + + let nonce = map.open(); + assert_eq!(0, nonce.instant); + drop(nonce); + + assert_eq!(map.get_seqno_safe_to_gc(), 0); + + let _ = global_seqno.next(); + + let nonce = map.open(); + assert_eq!(1, nonce.instant); + drop(nonce); + } + + #[test] + fn snapshot_tracker_increase_watermark() { + let global_seqno = SequenceNumberCounter::default(); + + let map = SnapshotTracker::new(global_seqno.clone()); + + // Simulates some tx committing + for _ in 0..100_000 { + let _ = global_seqno.next(); + let nonce = map.open(); + drop(nonce); + } + + assert!(map.get_seqno_safe_to_gc() > 0); + } + + #[test] + fn snapshot_tracker_prevent_watermark() { + let global_seqno = SequenceNumberCounter::default(); + + let map = SnapshotTracker::new(global_seqno.clone()); + + // This nonce prevents the watermark from increasing + let _old_nonce = map.open(); + + // Simlates some inserts happening + for _ in 0..1_000 { + let _ = global_seqno.next(); + } + + // Simulates more read tx opening and closing + for _ in 0..10_000 { + let nonce = map.open(); + drop(nonce); + } + + assert_eq!(map.get_seqno_safe_to_gc(), 0); + } + + #[test] + fn snapshot_tracker_close_never_opened_does_not_underflow_or_panic() { + let global_seqno = SequenceNumberCounter::default(); + let map = SnapshotTracker::new(global_seqno); + + assert_eq!(map.len(), 0); + map.close_raw(42); + assert_eq!(map.get_seqno_safe_to_gc(), 0); + } + + #[test] + fn snapshot_tracker_concurrent_open_same_seqno_counts_correctly() { + let global_seqno = SequenceNumberCounter::default(); + let map = SnapshotTracker::new(global_seqno); + + // make sure seqno doesn't change between two opens + let n1 = map.open(); + let n2 = map.open(); + assert_eq!(n1.instant, n2.instant); + assert_eq!(map.open_snapshots(), 2); + + // closing one decreases count + map.close(&n1); + assert_eq!(map.open_snapshots(), 1); + + // closing the other removes the last + map.close(&n2); + assert_eq!(map.open_snapshots(), 0); + } + + #[test] + fn snapshot_tracker_publish_moves_seqno_forward_and_ignores_older() { + let global_seqno = SequenceNumberCounter::default(); + let map = SnapshotTracker::new(global_seqno); + + let big = 100u64; + map.publish(big); + assert!(map.get() == (big + 1)); + + let before = map.get(); + map.publish(1); + assert_eq!(map.get(), before); + } + + #[test] + fn snapshot_tracker_clone_snapshot_behaves_like_second_open() { + let global_seqno = SequenceNumberCounter::default(); + let map = SnapshotTracker::new(global_seqno); + + let orig = map.open(); + let clone = map.clone_snapshot(&orig); + + assert_eq!(orig.instant, clone.instant); + assert_eq!(map.open_snapshots(), 2); + + // closing one leaves one + map.close(&orig); + assert_eq!(map.open_snapshots(), 1); + + // closing the clone removes all + map.close(&clone); + assert_eq!(map.open_snapshots(), 0); + } +} diff --git a/crates/fjall/src/stats.rs b/crates/fjall/src/stats.rs new file mode 100644 index 000000000..bc7270ddd --- /dev/null +++ b/crates/fjall/src/stats.rs @@ -0,0 +1,20 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::sync::atomic::{AtomicU64, AtomicUsize}; + +/// Ephemeral, runtime stats +#[derive(Default)] +pub struct Stats { + /// Active compaction conter + pub(crate) active_compaction_count: AtomicUsize, + + /// Time spent in compactions (in µs) + pub(crate) time_compacting: AtomicU64, + + /// Number of completed compactions + pub(crate) compactions_completed: AtomicUsize, +} + +impl Stats {} diff --git a/crates/fjall/src/supervisor.rs b/crates/fjall/src/supervisor.rs new file mode 100644 index 000000000..d29f530a0 --- /dev/null +++ b/crates/fjall/src/supervisor.rs @@ -0,0 +1,70 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use lsm_tree::SequenceNumberCounter; + +use crate::{ + db::Keyspaces, + flush::manager::FlushManager, + journal::{manager::JournalManager, Journal}, + snapshot_tracker::SnapshotTracker, + write_buffer_manager::WriteBufferManager, +}; +use std::sync::{Arc, RwLock}; + +pub struct SupervisorInner { + pub db_config: crate::Config, + + /// Dictionary of all keyspaces + #[doc(hidden)] + pub keyspaces: Arc>, + + pub(crate) write_buffer_size: WriteBufferManager, + pub(crate) flush_manager: FlushManager, + + pub seqno: SequenceNumberCounter, + + pub snapshot_tracker: SnapshotTracker, + + pub(crate) journal: Arc, + + /// Tracks journal size and garbage collects sealed journals when possible + pub(crate) journal_manager: Arc>, +} + +#[derive(Clone)] +pub struct Supervisor(Arc); + +impl std::ops::Deref for Supervisor { + type Target = SupervisorInner; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Supervisor { + pub fn new(inner: SupervisorInner) -> Self { + Self(Arc::new(inner)) + } + + pub fn build_seqno_map( + keyspaces: &Keyspaces, + ) -> Vec { + use crate::AbstractTree; + + let mut seqnos = Vec::with_capacity(keyspaces.len()); + + for keyspace in keyspaces.values() { + if let Some(lsn) = keyspace.tree.get_highest_memtable_seqno() { + seqnos.push(crate::journal::manager::EvictionWatermark { + lsn, + keyspace: keyspace.clone(), + }); + } + } + + seqnos + } +} diff --git a/crates/fjall/src/tx/mod.rs b/crates/fjall/src/tx/mod.rs new file mode 100644 index 000000000..4a0c61dfa --- /dev/null +++ b/crates/fjall/src/tx/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod optimistic; +pub mod single_writer; +pub mod write_tx; diff --git a/crates/fjall/src/tx/optimistic/conflict_manager.rs b/crates/fjall/src/tx/optimistic/conflict_manager.rs new file mode 100644 index 000000000..992d52fce --- /dev/null +++ b/crates/fjall/src/tx/optimistic/conflict_manager.rs @@ -0,0 +1,197 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::keyspace::InternalKeyspaceId; +use core::ops::Bound; +use lsm_tree::Slice; +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::RangeBounds, + sync::Mutex, +}; + +#[derive(Clone, Debug)] +enum Read { + Single(Slice), + Range { + start: Bound, + end: Bound, + }, + All, +} + +#[derive(Default, Debug)] +pub struct ConflictManager { + reads: Mutex>>, + conflict_keys: Mutex>>, +} + +impl ConflictManager { + fn push_read(&self, keyspace_id: InternalKeyspaceId, read: Read) { + #[expect(clippy::expect_used)] + let mut lock = self.reads.lock().expect("lock is poisoned"); + + if let Some(tbl) = lock.get_mut(&keyspace_id) { + tbl.push(read); + } else { + lock.entry(keyspace_id).or_default().push(read); + } + } + + pub fn mark_read(&self, keyspace_id: InternalKeyspaceId, key: Slice) { + self.push_read(keyspace_id, Read::Single(key)); + } + + pub fn mark_conflict(&self, keyspace_id: InternalKeyspaceId, key: Slice) { + #[expect(clippy::expect_used)] + let mut lock = self.conflict_keys.lock().expect("lock is poisoned"); + + if let Some(tbl) = lock.get_mut(&keyspace_id) { + tbl.insert(key); + } else { + lock.entry(keyspace_id).or_default().insert(key); + } + } + + pub fn mark_range(&self, keyspace_id: InternalKeyspaceId, range: impl RangeBounds) { + let start = match range.start_bound() { + Bound::Included(k) => Bound::Included(k.clone()), + Bound::Excluded(k) => Bound::Excluded(k.clone()), + Bound::Unbounded => Bound::Unbounded, + }; + + let end = match range.end_bound() { + Bound::Included(k) => Bound::Included(k.clone()), + Bound::Excluded(k) => Bound::Excluded(k.clone()), + Bound::Unbounded => Bound::Unbounded, + }; + + let read = if start == Bound::Unbounded && end == Bound::Unbounded { + Read::All + } else { + Read::Range { start, end } + }; + + self.push_read(keyspace_id, read); + } + + #[expect(clippy::too_many_lines, clippy::significant_drop_tightening)] + pub fn has_conflict(&self, other: &Self) -> bool { + #[expect(clippy::expect_used)] + let reads_lock = self.reads.lock().expect("lock is poisoned"); + + #[expect(clippy::expect_used)] + let conflict_keys_lock = other.conflict_keys.lock().expect("lock is poisoned"); + + if reads_lock.is_empty() { + return false; + } + + for (keyspace_name, keys) in &*reads_lock { + if let Some(other_conflict_keys) = conflict_keys_lock.get(keyspace_name) { + for ro in keys { + match ro { + Read::Single(k) => { + if other_conflict_keys.contains(k) { + return true; + } + } + Read::Range { start, end } => match (start, end) { + (Bound::Included(start), Bound::Included(end)) => { + if other_conflict_keys + .range::(( + Bound::Included(start), + Bound::Included(end), + )) + .next() + .is_some() + { + return true; + } + } + (Bound::Included(start), Bound::Excluded(end)) => { + if other_conflict_keys + .range::(( + Bound::Included(start), + Bound::Excluded(end), + )) + .next() + .is_some() + { + return true; + } + } + (Bound::Included(start), Bound::Unbounded) => { + if other_conflict_keys + .range::((Bound::Included(start), Bound::Unbounded)) + .next() + .is_some() + { + return true; + } + } + (Bound::Excluded(start), Bound::Included(end)) => { + if other_conflict_keys + .range::(( + Bound::Excluded(start), + Bound::Included(end), + )) + .next() + .is_some() + { + return true; + } + } + (Bound::Excluded(start), Bound::Excluded(end)) => { + if other_conflict_keys + .range::(( + Bound::Excluded(start), + Bound::Excluded(end), + )) + .next() + .is_some() + { + return true; + } + } + (Bound::Excluded(start), Bound::Unbounded) => { + if other_conflict_keys + .range::((Bound::Excluded(start), Bound::Unbounded)) + .next() + .is_some() + { + return true; + } + } + (Bound::Unbounded, Bound::Included(end)) => { + let range = ..=end; + for write in other_conflict_keys { + if range.contains(&write) { + return true; + } + } + } + (Bound::Unbounded, Bound::Excluded(end)) => { + let range = ..end; + for write in other_conflict_keys { + if range.contains(&write) { + return true; + } + } + } + (Bound::Unbounded, Bound::Unbounded) => unreachable!(), + }, + Read::All => { + if !other_conflict_keys.is_empty() { + return true; + } + } + } + } + } + } + + false + } +} diff --git a/crates/fjall/src/tx/optimistic/keyspace.rs b/crates/fjall/src/tx/optimistic/keyspace.rs new file mode 100644 index 000000000..3e9d003a3 --- /dev/null +++ b/crates/fjall/src/tx/optimistic/keyspace.rs @@ -0,0 +1,502 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{tx::optimistic::OptimisticTxDatabase, Keyspace}; +use crate::{Guard, Readable}; +use lsm_tree::{UserKey, UserValue}; +use std::path::PathBuf; + +/// Handle to a keyspace of a transactional database +#[derive(Clone)] +pub struct OptimisticTxKeyspace { + pub(crate) inner: Keyspace, + pub(crate) db: OptimisticTxDatabase, +} + +impl AsRef for OptimisticTxKeyspace { + fn as_ref(&self) -> &Keyspace { + self.inner() + } +} + +impl OptimisticTxKeyspace { + /// Returns the underlying LSM-tree's path. + #[must_use] + pub fn path(&self) -> PathBuf { + self.inner.path().into() + } + + /// Approximates the number of items in the keyspace. + /// + /// For update- or delete-heavy workloads, this value will + /// diverge from the real value, but is a O(1) operation. + /// + /// For insert-only workloads (e.g. logs, time series) + /// this value is reliable. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert_eq!(tree.approximate_len(), 0); + /// + /// tree.insert("1", "abc")?; + /// assert_eq!(tree.approximate_len(), 1); + /// + /// tree.remove("1")?; + /// // Oops! approximate_len will not be reliable here + /// assert_eq!(tree.approximate_len(), 2); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn approximate_len(&self) -> usize { + self.inner.approximate_len() + } + + /// Removes an item and returns its value if it existed. + /// + /// The operation will run wrapped in a transaction. + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let taken = tree.take("a")?.unwrap(); + /// assert_eq!(b"abc", &*taken); + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn take>(&self, key: K) -> crate::Result> { + self.fetch_update(key, |_| None) + } + + /// Atomically updates an item and returns the previous value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Note + /// + /// The provided closure can be called multiple times as this function + /// automatically retries on conflict. Since this is an `FnMut`, make sure + /// it is idempotent and will not cause side-effects. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, Slice, KeyspaceCreateOptions}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let prev = tree.fetch_update("a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let prev = tree.fetch_update("a", |_| None)?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn fetch_update, F: FnMut(Option<&UserValue>) -> Option>( + &self, + key: K, + mut f: F, + ) -> crate::Result> { + let key = key.into(); + + loop { + let mut tx = self.db.write_tx()?; + let prev = tx.fetch_update(self.inner(), key.clone(), &mut f)?; + if tx.commit()?.is_ok() { + return Ok(prev); + } + } + } + + /// Atomically updates an item and returns the new value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Note + /// + /// The provided closure can be called multiple times as this function + /// automatically retries on conflict. Since this is an `FnMut`, make sure + /// it is idempotent and will not cause side-effects. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, Slice, KeyspaceCreateOptions}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let updated = tree.update_fetch("a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"def", &*updated); + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let updated = tree.update_fetch("a", |_| None)?; + /// assert!(updated.is_none()); + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn update_fetch, F: FnMut(Option<&UserValue>) -> Option>( + &self, + key: K, + mut f: F, + ) -> crate::Result> { + let key = key.into(); + + loop { + let mut tx = self.db.write_tx()?; + let updated = tx.update_fetch(self.inner(), key.clone(), &mut f)?; + if tx.commit()?.is_ok() { + return Ok(updated); + } + } + } + + /// Inserts a key-value pair into the keyspace. + /// + /// Keys may be up to 65536 bytes long, values up to 2^32 bytes. + /// Shorter keys and values result in better performance. + /// + /// If the key already exists, the item will be overwritten. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// assert!(!db.read_tx().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn insert, V: Into>( + &self, + key: K, + value: V, + ) -> crate::Result<()> { + let mut tx = self.db.write_tx()?; + tx.insert(self.inner(), key, value); + + #[expect( + clippy::expect_used, + clippy::missing_panics_doc, + reason = "blind insert should not conflict ever" + )] + tx.commit()?.expect("blind insert should not conflict ever"); + + Ok(()) + } + + /// Removes an item from the keyspace. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// assert!(!db.read_tx().is_empty(&tree)?); + /// + /// tree.remove("a")?; + /// assert!(db.read_tx().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn remove>(&self, key: K) -> crate::Result<()> { + let mut tx = self.db.write_tx()?; + tx.remove(self.inner(), key); + + #[expect( + clippy::expect_used, + clippy::missing_panics_doc, + reason = "blind remove should not conflict ever" + )] + tx.commit()?.expect("blind remove should not conflict ever"); + + Ok(()) + } + + /// Removes an item from the keyspace, leaving behind a weak tombstone. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Experimental + /// + /// This function is currently experimental. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// assert!(!db.read_tx().is_empty(&tree)?); + /// + /// tree.remove_weak("a")?; + /// assert!(db.read_tx().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn remove_weak>(&self, key: K) -> crate::Result<()> { + let mut tx = self.db.write_tx()?; + tx.remove_weak(self.inner(), key); + + #[expect(clippy::expect_used)] + tx.commit()?.expect("blind remove should not conflict ever"); + + Ok(()) + } + + /// Retrieves an item from the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("my_value".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn get>(&self, key: K) -> crate::Result> { + self.inner.get(key) + } + + /// Retrieves the size of an item from the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let len = tree.size_of("a")?.unwrap_or_default(); + /// assert_eq!("my_value".len() as u32, len); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn size_of>(&self, key: K) -> crate::Result> { + self.inner.size_of(key) + } + + /// Returns the first key-value pair in the keyspace. + /// The key in this pair is the minimum key in the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// tree.insert("b", "my_value")?; + /// + /// assert_eq!(b"a", &*tree.first_key_value().unwrap().key()?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn first_key_value(&self) -> Option { + let read_tx = self.db.read_tx(); + read_tx.first_key_value(self) + } + + /// Returns the last key-value pair in the keyspace. + /// The key in this pair is the maximum key in the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// tree.insert("b", "my_value")?; + /// + /// assert_eq!(b"b", &*tree.last_key_value().unwrap().key()?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn last_key_value(&self) -> Option { + let read_tx = self.db.read_tx(); + read_tx.last_key_value(self) + } + + /// Returns `true` if the keyspace contains the specified key. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// assert!(tree.contains_key("a")?); + /// assert!(!tree.contains_key("b")?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn contains_key>(&self, key: K) -> crate::Result { + self.inner.contains_key(key) + } + + /// Allows access to the inner keyspace handle, allowing to + /// escape from the transactional context. + #[doc(hidden)] + #[must_use] + pub fn inner(&self) -> &Keyspace { + &self.inner + } +} diff --git a/crates/fjall/src/tx/optimistic/mod.rs b/crates/fjall/src/tx/optimistic/mod.rs new file mode 100644 index 000000000..1dca25162 --- /dev/null +++ b/crates/fjall/src/tx/optimistic/mod.rs @@ -0,0 +1,184 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod conflict_manager; +mod keyspace; +mod oracle; +mod write_tx; + +use crate::{ + keyspace::KeyspaceKey, + tx::{optimistic::oracle::Oracle, single_writer::Openable}, + Config, Database, KeyspaceCreateOptions, PersistMode, Snapshot, +}; +use std::{ + path::Path, + sync::{Arc, Mutex}, +}; + +pub use keyspace::OptimisticTxKeyspace; +pub use write_tx::{Conflict, WriteTransaction}; + +/// Transactional database +#[derive(Clone)] +pub struct OptimisticTxDatabase { + pub(crate) inner: Database, + pub(super) oracle: Arc, +} + +impl Openable for OptimisticTxDatabase { + fn open(config: Config) -> crate::Result + where + Self: Sized, + { + let inner = Database::create_or_recover(config)?; + // inner.start_background_threads()?; + + Ok(Self { + oracle: Arc::new(Oracle { + write_serialize_lock: Mutex::default(), + snapshot_tracker: inner.supervisor.snapshot_tracker.clone(), + }), + inner, + }) + } +} + +impl OptimisticTxDatabase { + /// Creates a new database builder to create or open a database at `path`. + pub fn builder(path: impl AsRef) -> crate::DatabaseBuilder { + crate::DatabaseBuilder::new(path.as_ref()) + } + + #[doc(hidden)] + #[must_use] + pub fn inner(&self) -> &Database { + &self.inner + } + + /// Starts a new writeable transaction. + /// + /// # Errors + /// + /// Will return `Err` if creation failed. + pub fn write_tx(&self) -> crate::Result { + let snapshot = { + // acquire a lock here to prevent getting a stale snapshot seqno + // this will drain at least part of the commit queue, but ordering + // is platform-dependent since we use std::sync::Mutex + let _guard = self.oracle.write_serialize_lock()?; + + self.inner.supervisor.snapshot_tracker.open() + }; + + let mut write_tx = + WriteTransaction::new(self.inner().clone(), snapshot, self.oracle.clone()); + + if !self.inner.config.manual_journal_persist { + write_tx = write_tx.durability(Some(PersistMode::Buffer)); + } + + Ok(write_tx) + } + + /// Starts a new read-only transaction (a.k.a. [`Snapshot`]). + #[must_use] + pub fn read_tx(&self) -> Snapshot { + self.inner.snapshot() + } + + /// Flushes the active journal. The durability depends on the [`PersistMode`] + /// used. + /// + /// Persisting only affects durability, NOT consistency! Even without flushing + /// data is crash-safe. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{PersistMode, OptimisticTxDatabase, KeyspaceCreateOptions}; + /// # let folder = tempfile::tempdir()?; + /// let db = OptimisticTxDatabase::builder(folder).open()?; + /// let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + /// + /// items.insert("a", "hello")?; + /// + /// db.persist(PersistMode::SyncAll)?; + /// # + /// # Ok::<_, fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn persist(&self, mode: PersistMode) -> crate::Result<()> { + self.inner.persist(mode) + } + + /// Creates or opens a keyspace. + /// + /// If the keyspace does not yet exist, it will be created configured with `create_options`. + /// Otherwise simply a handle to the existing keyspace will be returned. + /// + /// Keyspace names can be up to 255 characters long and can not be empty. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + /// + /// # Panics + /// + /// Panics if the keyspace name is invalid. + pub fn keyspace( + &self, + name: &str, + create_options: impl FnOnce() -> KeyspaceCreateOptions, + ) -> crate::Result { + let keyspace = self.inner.keyspace(name, create_options)?; + + Ok(OptimisticTxKeyspace { + inner: keyspace, + db: self.clone(), + }) + } + + /// Returns the number of keyspaces. + #[must_use] + pub fn keyspace_count(&self) -> usize { + self.inner.keyspace_count() + } + + /// Gets a list of all keyspace names in the database. + #[must_use] + pub fn list_keyspace_names(&self) -> Vec { + self.inner.list_keyspace_names() + } + + /// Returns `true` if the keyspace with the given name exists. + #[must_use] + pub fn keyspace_exists(&self, name: &str) -> bool { + self.inner.keyspace_exists(name) + } + + /// Returns the current write buffer size (active + sealed memtables). + #[must_use] + pub fn write_buffer_size(&self) -> u64 { + self.inner.write_buffer_size() + } + + /// Returns the number of journal fragments on disk. + #[must_use] + pub fn journal_count(&self) -> usize { + self.inner.journal_count() + } + + /// Returns the disk space usage of the entire database. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn disk_space(&self) -> crate::Result { + self.inner.disk_space() + } +} diff --git a/crates/fjall/src/tx/optimistic/oracle.rs b/crates/fjall/src/tx/optimistic/oracle.rs new file mode 100644 index 000000000..5b423a6c1 --- /dev/null +++ b/crates/fjall/src/tx/optimistic/oracle.rs @@ -0,0 +1,121 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::conflict_manager::ConflictManager; +use crate::snapshot_tracker::SnapshotTracker; +use crate::SeqNo; +use std::collections::BTreeMap; +use std::sync::{Mutex, MutexGuard}; + +pub enum CommitOutcome { + Ok, + Aborted(E), + Conflicted, +} + +pub struct Oracle { + pub(super) write_serialize_lock: Mutex>, + pub(super) snapshot_tracker: SnapshotTracker, +} + +impl Oracle { + pub(super) fn with_commit Result<(), E>>( + &self, + instant: SeqNo, + conflict_checker: ConflictManager, + f: F, + ) -> crate::Result> { + let mut committed_txns = self + .write_serialize_lock + .lock() + .map_err(|_| crate::Error::Poisoned)?; + + // If the committed_txn.ts is less than `SeqNo` that implies that the + // committed_txn finished before the current transaction started. + // We don't need to check for conflict in that case. + // This change assumes linearizability. Lack of linearizability could + // cause the read ts of a new txn to be lower than the commit ts of + // a txn before it. + let conflicted = + committed_txns + .range((instant + 1)..) + .any(|(_ts, other_conflict_checker)| { + conflict_checker.has_conflict(other_conflict_checker) + }); + + self.snapshot_tracker.close_raw(instant); + + // TODO: This can be expensive and should probably be done in a background worker, or a after a memtable rotation + let safe_to_gc = self.snapshot_tracker.get_seqno_safe_to_gc(); + committed_txns.retain(|ts, _| *ts > safe_to_gc); + + if conflicted { + return Ok(CommitOutcome::Conflicted); + } + + if let Err(e) = f() { + return Ok(CommitOutcome::Aborted(e)); + } + + committed_txns.insert(self.snapshot_tracker.get(), conflict_checker); + + Ok(CommitOutcome::Ok) + } + + pub(super) fn write_serialize_lock( + &self, + ) -> crate::Result>> { + self.write_serialize_lock + .lock() + .map_err(|_| crate::Error::Poisoned) + } +} + +#[cfg(test)] +mod tests { + use crate::{KeyspaceCreateOptions, OptimisticTxDatabase, OptimisticTxKeyspace, Readable}; + + #[expect(clippy::significant_drop_tightening)] + fn run_tx( + db: &OptimisticTxDatabase, + tree: &OptimisticTxKeyspace, + ) -> Result<(), Box> { + let mut tx1 = db.write_tx()?; + let mut tx2 = db.write_tx()?; + tx1.insert(tree.inner(), "hello", "world"); + + tx1.commit()??; + assert!(tree.contains_key("hello")?); + + _ = tx2.get(tree.inner(), "hello")?; + + tx2.insert(tree.inner(), "hello", "world2"); + assert!(tx2.commit()?.is_err()); // intended to conflict + + Ok(()) + } + + #[expect(clippy::unwrap_used)] + #[test] + fn oracle_committed_txns_does_not_leak() -> crate::Result<()> { + let tmpdir = tempfile::tempdir()?; + let db = OptimisticTxDatabase::builder(tmpdir.path()).open()?; + + let part = db.keyspace("foo", KeyspaceCreateOptions::default)?; + + for _ in 0..10_000 { + run_tx(&db, &part).unwrap(); + } + + assert!(dbg!(db.oracle.write_serialize_lock.lock().unwrap().len()) < 10_000); + + for _ in 0..10_000 { + run_tx(&db, &part).unwrap(); + } + + assert!(dbg!(db.oracle.write_serialize_lock.lock().unwrap().len()) < 10_000); + + Ok(()) + } +} diff --git a/crates/fjall/src/tx/optimistic/write_tx.rs b/crates/fjall/src/tx/optimistic/write_tx.rs new file mode 100644 index 000000000..e4cfc2dfc --- /dev/null +++ b/crates/fjall/src/tx/optimistic/write_tx.rs @@ -0,0 +1,891 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + snapshot_nonce::SnapshotNonce, + tx::{ + optimistic::{ + conflict_manager::ConflictManager, + oracle::{CommitOutcome, Oracle}, + }, + write_tx::BaseTransaction, + }, + Database, Guard, Iter, Keyspace, PersistMode, Readable, +}; +use lsm_tree::{Slice, UserKey, UserValue}; +use std::{ + fmt, + ops::{Bound, RangeBounds, RangeFull}, + sync::Arc, +}; + +/// Transaction conflict +/// +/// SSI transactions can conflict which require them to be rerun. +#[derive(Debug)] +pub struct Conflict; + +impl std::error::Error for Conflict {} + +impl fmt::Display for Conflict { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + "Transaction conflict".fmt(f) + } +} + +/// A cross-keyspace transaction using optimistic concurrency control +/// +/// Use [`WriteTransaction::commit`] to commit changes to the keyspace(s). +/// +/// Transactions keep a consistent view of the database at the time, +/// meaning old data will not be dropped until it is not referenced by any active transaction. +/// +/// For that reason, you should try to keep transactions short-lived, and make sure they +/// are not held somewhere forever. +/// +/// # Caution +/// +/// The transaction may fail and have to be rerun if it conflicts. +#[clippy::has_significant_drop] +pub struct WriteTransaction { + inner: BaseTransaction, + cm: ConflictManager, + oracle: Arc, +} + +impl Readable for WriteTransaction { + fn get>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + let keyspace = keyspace.as_ref(); + + let res = self.inner.get(keyspace, key.as_ref())?; + + self.cm.mark_read(keyspace.id, key.as_ref().into()); + + Ok(res) + } + + fn contains_key>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result { + let keyspace = keyspace.as_ref(); + + let contains = self.inner.contains_key(keyspace, key.as_ref())?; + + self.cm.mark_read(keyspace.id, key.as_ref().into()); + + Ok(contains) + } + + fn first_key_value(&self, keyspace: impl AsRef) -> Option { + self.iter(&keyspace).next() + } + + fn last_key_value(&self, keyspace: impl AsRef) -> Option { + self.iter(&keyspace).next_back() + } + + fn size_of>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + let keyspace = keyspace.as_ref(); + self.inner.size_of(keyspace, key) + } + + fn iter(&self, keyspace: impl AsRef) -> Iter { + self.cm.mark_range(keyspace.as_ref().id, RangeFull); + self.inner.iter(keyspace) + } + + fn range, R: RangeBounds>( + &self, + keyspace: impl AsRef, + range: R, + ) -> Iter { + let start: Bound = range.start_bound().map(|k| k.as_ref().into()); + let end: Bound = range.end_bound().map(|k| k.as_ref().into()); + + self.cm.mark_range(keyspace.as_ref().id, (start, end)); + + self.inner.range(keyspace, range) + } + + fn prefix>(&self, keyspace: impl AsRef, prefix: K) -> Iter { + self.range(keyspace, lsm_tree::range::prefix_to_range(prefix.as_ref())) + } +} + +impl WriteTransaction { + pub(crate) fn new(db: Database, nonce: SnapshotNonce, oracle: Arc) -> Self { + Self { + inner: BaseTransaction::new(db, nonce), + cm: ConflictManager::default(), + oracle, + } + } + + /// Sets the durability level. + #[must_use] + pub fn durability(mut self, mode: Option) -> Self { + self.inner = self.inner.durability(mode); + self + } + + /// Removes an item and returns its value if it existed. + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx()?; + /// + /// let taken = tx.take(&tree, "a")?.unwrap(); + /// assert_eq!(b"abc", &*taken); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn take>( + &mut self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + self.fetch_update(keyspace, key, |_| None) + } + + /// Atomically updates an item and returns the new value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable, Slice}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx()?; + /// + /// let updated = tx.update_fetch(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"def", &*updated); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx()?; + /// + /// let updated = tx.update_fetch(&tree, "a", |_| None)?; + /// assert!(updated.is_none()); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn update_fetch, F: FnOnce(Option<&UserValue>) -> Option>( + &mut self, + keyspace: impl AsRef, + key: K, + f: F, + ) -> crate::Result> { + let keyspace = keyspace.as_ref(); + let key: UserKey = key.into(); + + let updated = self.inner.update_fetch(keyspace, key.clone(), f)?; + + self.cm.mark_read(keyspace.id, key.clone()); + self.cm.mark_conflict(keyspace.id, key); + + Ok(updated) + } + + /// Atomically updates an item and returns the previous value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable, Slice}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx()?; + /// + /// let prev = tx.fetch_update(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx()?; + /// + /// let prev = tx.fetch_update(&tree, "a", |_| None)?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn fetch_update, F: FnOnce(Option<&UserValue>) -> Option>( + &mut self, + keyspace: impl AsRef, + key: K, + f: F, + ) -> crate::Result> { + let keyspace = keyspace.as_ref(); + let key = key.into(); + + let prev = self.inner.fetch_update(keyspace, key.clone(), f)?; + + self.cm.mark_read(keyspace.id, key.clone()); + self.cm.mark_conflict(keyspace.id, key); + + Ok(prev) + } + + /// Inserts a key-value pair into the keyspace. + /// + /// Keys may be up to 65536 bytes long, values up to 2^32 bytes. + /// Shorter keys and values result in better performance. + /// + /// If the key already exists, the item will be overwritten. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "previous_value")?; + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// + /// let mut tx = db.write_tx()?; + /// tx.insert(&tree, "a", "new_value"); + /// + /// drop(tx); + /// + /// // Write was not committed + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn insert, V: Into>( + &mut self, + keyspace: impl AsRef, + key: K, + value: V, + ) { + let keyspace = keyspace.as_ref(); + let key: UserKey = key.into(); + + self.inner.insert(keyspace, key.clone(), value); + self.cm.mark_conflict(keyspace.id, key); + } + + /// Removes an item from the keyspace. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "previous_value")?; + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// + /// let mut tx = db.write_tx()?; + /// tx.remove(&tree, "a"); + /// + /// // Read-your-own-write + /// let item = tx.get(&tree, "a")?; + /// assert_eq!(None, item); + /// + /// drop(tx); + /// + /// // Deletion was not committed + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn remove>(&mut self, keyspace: impl AsRef, key: K) { + let keyspace = keyspace.as_ref(); + let key: UserKey = key.into(); + + self.inner.remove(keyspace, key.clone()); + self.cm.mark_conflict(keyspace.id, key); + } + + /// Removes an item from the keyspace, leaving behind a weak tombstone. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// # Experimental + /// + /// This function is currently experimental. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{OptimisticTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = OptimisticTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "previous_value")?; + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// + /// let mut tx = db.write_tx()?; + /// tx.remove_weak(&tree, "a"); + /// + /// // Read-your-own-write + /// let item = tx.get(&tree, "a")?; + /// assert_eq!(None, item); + /// + /// drop(tx); + /// + /// // Deletion was not committed + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[doc(hidden)] + pub fn remove_weak>(&mut self, keyspace: impl AsRef, key: K) { + let keyspace = keyspace.as_ref(); + let key: UserKey = key.into(); + + self.inner.remove_weak(keyspace, key.clone()); + self.cm.mark_conflict(keyspace.id, key); + } + + /// Commits the transaction. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn commit(self) -> crate::Result> { + // NOTE: We have no write set, so we are basically + // a read-only transaction, so nothing to do here + if self.inner.memtables.is_empty() { + return Ok(Ok(())); + } + + let oracle = self.oracle.clone(); + + match oracle.with_commit(self.inner.nonce.instant, self.cm, move || { + self.inner.commit() + })? { + CommitOutcome::Ok => Ok(Ok(())), + CommitOutcome::Aborted(e) => Err(e), + CommitOutcome::Conflicted => Ok(Err(Conflict)), + } + } + + /// More explicit alternative to dropping the transaction + /// to roll it back. + pub fn rollback(self) { + self.inner.rollback(); + } +} + +#[cfg(test)] +mod tests { + use crate::{ + Conflict, KeyspaceCreateOptions, OptimisticTxDatabase, OptimisticTxKeyspace, Readable, + }; + use tempfile::TempDir; + use test_log::test; + + struct TestEnv { + db: OptimisticTxDatabase, + tree: OptimisticTxKeyspace, + + #[expect(unused)] + tmpdir: TempDir, + } + + impl TestEnv { + fn seed_hermitage_data(&self) -> crate::Result<()> { + self.tree.insert([1u8], [10u8])?; + self.tree.insert([2u8], [20u8])?; + Ok(()) + } + } + + fn setup() -> Result> { + let tmpdir = tempfile::tempdir()?; + let db = OptimisticTxDatabase::builder(tmpdir.path()).open()?; + + let tree = db.keyspace("foo", KeyspaceCreateOptions::default)?; + + Ok(TestEnv { db, tree, tmpdir }) + } + + // Adapted from https://github.com/al8n/skipdb/issues/10 + #[test] + #[expect(clippy::unwrap_used)] + fn tx_ssi_arthur_1() -> Result<(), Box> { + let env = setup()?; + + let mut tx = env.db.write_tx()?; + tx.insert(env.tree.inner(), "a1", 10u64.to_be_bytes()); + tx.insert(env.tree.inner(), "b1", 100u64.to_be_bytes()); + tx.insert(env.tree.inner(), "b2", 200u64.to_be_bytes()); + tx.commit()??; + + let mut tx1 = env.db.write_tx()?; + let val = tx1 + .range(&env.tree, "a".."b") + .map(|kv| { + let v = kv.value().unwrap(); + + let mut buf = [0u8; 8]; + buf.copy_from_slice(&v); + u64::from_be_bytes(buf) + }) + .sum::(); + tx1.insert(env.tree.inner(), "b3", 10u64.to_be_bytes()); + assert_eq!(10, val); + + let mut tx2 = env.db.write_tx()?; + let val = tx2 + .range(&env.tree, "b".."c") + .map(|kv| { + let v = kv.value().unwrap(); + + let mut buf = [0u8; 8]; + buf.copy_from_slice(&v); + u64::from_be_bytes(buf) + }) + .sum::(); + tx2.insert(env.tree.inner(), "a3", 300u64.to_be_bytes()); + assert_eq!(300, val); + tx2.commit()??; + assert!(matches!(tx1.commit()?, Err(Conflict))); + + let tx3 = env.db.write_tx()?; + let val = tx3 + .iter(&env.tree) + .filter_map(|kv| { + let (k, v) = kv.into_inner().unwrap(); + + if k.starts_with(b"a") { + let mut buf = [0u8; 8]; + buf.copy_from_slice(&v); + Some(u64::from_be_bytes(buf)) + } else { + None + } + }) + .sum::(); + assert_eq!(310, val); + + Ok(()) + } + + // Adapted from https://github.com/al8n/skipdb/issues/10 + #[test] + #[expect(clippy::unwrap_used)] + fn tx_ssi_arthur_2() -> Result<(), Box> { + let env = setup()?; + + let mut tx = env.db.write_tx()?; + tx.insert(env.tree.inner(), "b1", 100u64.to_be_bytes()); + tx.insert(env.tree.inner(), "b2", 200u64.to_be_bytes()); + tx.commit()??; + + let mut tx1 = env.db.write_tx()?; + let val = tx1 + .range(&env.tree, "a".."b") + .map(|kv| { + let v = kv.value().unwrap(); + + let mut buf = [0u8; 8]; + buf.copy_from_slice(&v); + u64::from_be_bytes(buf) + }) + .sum::(); + tx1.insert(env.tree.inner(), "b3", 0u64.to_be_bytes()); + assert_eq!(0, val); + + let mut tx2 = env.db.write_tx()?; + let val = tx2 + .range(&env.tree, "b".."c") + .map(|kv| { + let v = kv.value().unwrap(); + + let mut buf = [0u8; 8]; + buf.copy_from_slice(&v); + u64::from_be_bytes(buf) + }) + .sum::(); + tx2.insert(env.tree.inner(), "a3", 300u64.to_be_bytes()); + assert_eq!(300, val); + tx2.commit()??; + assert!(matches!(tx1.commit()?, Err(Conflict))); + + let tx3 = env.db.write_tx()?; + let val = tx3 + .iter(&env.tree) + .filter_map(|kv| { + let (k, v) = kv.into_inner().unwrap(); + + if k.starts_with(b"a") { + let mut buf = [0u8; 8]; + buf.copy_from_slice(&v); + Some(u64::from_be_bytes(buf)) + } else { + None + } + }) + .sum::(); + assert_eq!(300, val); + + Ok(()) + } + + #[test] + fn tx_ssi_basic() -> Result<(), Box> { + let env = setup()?; + + let mut tx1 = env.db.write_tx()?; + let mut tx2 = env.db.write_tx()?; + + tx1.insert(env.tree.inner(), "hello", "world"); + + tx1.commit()??; + assert!(env.tree.contains_key("hello")?); + + assert_eq!(tx2.get(env.tree.inner(), "hello")?, None); + + tx2.insert(env.tree.inner(), "hello", "world2"); + assert!(matches!(tx2.commit()?, Err(Conflict))); + + let mut tx1 = env.db.write_tx()?; + let mut tx2 = env.db.write_tx()?; + + tx1.iter(&env.tree).next(); + tx2.insert(env.tree.inner(), "hello", "world2"); + + tx1.insert(env.tree.inner(), "hello2", "world1"); + tx1.commit()??; + + tx2.commit()??; + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn tx_ssi_ww() -> Result<(), Box> { + // https://en.wikipedia.org/wiki/Write%E2%80%93write_conflict + let env = setup()?; + + let mut tx1 = env.db.write_tx()?; + let mut tx2 = env.db.write_tx()?; + + tx1.insert(env.tree.inner(), "a", "a"); + tx2.insert(env.tree.inner(), "b", "c"); + tx1.insert(env.tree.inner(), "b", "b"); + tx1.commit()??; + + tx2.insert(env.tree.inner(), "a", "c"); + + tx2.commit()??; + assert_eq!(b"c", &*env.tree.get("a")?.unwrap()); + assert_eq!(b"c", &*env.tree.get("b")?.unwrap()); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn tx_ssi_swap() -> Result<(), Box> { + let env = setup()?; + + env.tree.insert("x", "x")?; + env.tree.insert("y", "y")?; + + let mut tx1 = env.db.write_tx()?; + let mut tx2 = env.db.write_tx()?; + + { + let x = tx1.get(env.tree.inner(), "x")?.unwrap(); + tx1.insert(env.tree.inner(), "y", x); + } + + { + let y = tx2.get(env.tree.inner(), "y")?.unwrap(); + tx2.insert(env.tree.inner(), "x", y); + } + + tx1.commit()??; + assert!(matches!(tx2.commit()?, Err(Conflict))); + + Ok(()) + } + + #[test] + fn tx_ssi_write_cycles() -> Result<(), Box> { + let env = setup()?; + env.seed_hermitage_data()?; + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + t1.insert(env.tree.inner(), [1u8], [11u8]); + t2.insert(env.tree.inner(), [1u8], [12u8]); + t1.insert(env.tree.inner(), [2u8], [21u8]); + t1.commit()??; + + assert_eq!(env.tree.get([1u8])?, Some([11u8].into())); + + t2.insert(env.tree.inner(), [2u8], [22u8]); + t2.commit()??; + + assert_eq!(env.tree.get([1u8])?, Some([12u8].into())); + assert_eq!(env.tree.get([2u8])?, Some([22u8].into())); + + Ok(()) + } + + #[test] + fn tx_ssi_aborted_reads() -> Result<(), Box> { + let env = setup()?; + env.seed_hermitage_data()?; + + let mut t1 = env.db.write_tx()?; + let t2 = env.db.write_tx()?; + + t1.insert(env.tree.inner(), [1u8], [101u8]); + + assert_eq!(t2.get(env.tree.inner(), [1u8])?, Some([10u8].into())); + + t1.rollback(); + + assert_eq!(t2.get(env.tree.inner(), [1u8])?, Some([10u8].into())); + + t2.commit()??; + + Ok(()) + } + + #[expect(clippy::unwrap_used)] + #[test] + fn tx_ssi_anti_dependency_cycles() -> Result<(), Box> { + let env = setup()?; + env.seed_hermitage_data()?; + + let mut t1 = env.db.write_tx()?; + { + let mut iter = t1.iter(&env.tree); + assert_eq!( + iter.next().unwrap().into_inner()?, + ([1u8].into(), [10u8].into()), + ); + assert_eq!( + iter.next().unwrap().into_inner()?, + ([2u8].into(), [20u8].into()), + ); + assert!(iter.next().is_none()); + } + + let mut t2 = env.db.write_tx()?; + let new = t2.update_fetch(&env.tree, [2u8], |v| { + v.and_then(|v| v.first().copied()).map(|v| [v + 5].into()) + })?; + assert_eq!(new, Some([25u8].into())); + t2.commit()??; + + let t3 = env.db.write_tx()?; + { + let mut iter = t3.iter(&env.tree); + assert_eq!( + iter.next().unwrap().into_inner()?, + ([1u8].into(), [10u8].into()), + ); + assert_eq!( + iter.next().unwrap().into_inner()?, + ([2u8].into(), [25u8].into()), + ); // changed here + assert!(iter.next().is_none()); + } + + t3.commit()??; + + t1.insert(env.tree.inner(), [1u8], [0u8]); + + assert!(matches!(t1.commit()?, Err(Conflict))); + + Ok(()) + } + + #[test] + fn tx_ssi_update_fetch_update() -> Result<(), Box> { + let env = setup()?; + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + let new = t1.update_fetch(env.tree.inner(), "hello", |_| Some("world".into()))?; + assert_eq!(new, Some("world".into())); + let old = t2.fetch_update(env.tree.inner(), "hello", |_| Some("world2".into()))?; + assert_eq!(old, None); + + t1.commit()??; + assert!(matches!(t2.commit()?, Err(Conflict))); + + assert_eq!(env.tree.get("hello")?, Some("world".into())); + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + let old = t1.fetch_update(env.tree.inner(), "hello", |_| Some("world3".into()))?; + assert_eq!(old, Some("world".into())); + let new = t2.update_fetch(env.tree.inner(), "hello2", |_| Some("world2".into()))?; + assert_eq!(new, Some("world2".into())); + + t1.commit()??; + t2.commit()??; + + assert_eq!(env.tree.get("hello")?, Some("world3".into())); + assert_eq!(env.tree.get("hello2")?, Some("world2".into())); + + Ok(()) + } + + #[test] + fn tx_ssi_range() -> Result<(), Box> { + let env = setup()?; + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + _ = t1.range(&env.tree, "h"..="hello"); + t1.insert(env.tree.inner(), "foo", "bar"); + + // insert a key INSIDE the range read by t1 + t2.insert(env.tree.inner(), "hello", "world"); + + t2.commit()??; + assert!(matches!(t1.commit()?, Err(Conflict))); + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + _ = t1.range(&env.tree, "h"..="hello"); + t1.insert(env.tree.inner(), "foo", "bar"); + + // insert a key OUTSIDE the range read by t1 + t2.insert(env.tree.inner(), "hello2", "world"); + + t2.commit()??; + t1.commit()??; + + Ok(()) + } + + #[test] + fn tx_ssi_prefix() -> Result<(), Box> { + let env = setup()?; + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + _ = t1.prefix(&env.tree, "hello"); + t1.insert(env.tree.inner(), "foo", "bar"); + + // insert a key MATCHING the prefix read by t1 + t2.insert(env.tree.inner(), "hello", "world"); + + t2.commit()??; + assert!(matches!(t1.commit()?, Err(Conflict))); + + let mut t1 = env.db.write_tx()?; + let mut t2 = env.db.write_tx()?; + + _ = t1.prefix(&env.tree, "hello"); + t1.insert(env.tree.inner(), "foo", "bar"); + + // insert a key NOT MATCHING the range read by t1 + t2.insert(env.tree.inner(), "foobar", "world"); + + t2.commit()??; + t1.commit()??; + + Ok(()) + } +} diff --git a/crates/fjall/src/tx/single_writer/keyspace.rs b/crates/fjall/src/tx/single_writer/keyspace.rs new file mode 100644 index 000000000..ab0ac3179 --- /dev/null +++ b/crates/fjall/src/tx/single_writer/keyspace.rs @@ -0,0 +1,469 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{Guard, Keyspace, Readable, SingleWriterTxDatabase}; +use lsm_tree::{UserKey, UserValue}; +use std::path::PathBuf; + +/// Handle to a keyspace of a transactional database +#[derive(Clone)] +pub struct SingleWriterTxKeyspace { + pub(crate) inner: Keyspace, + pub(crate) db: SingleWriterTxDatabase, +} + +impl AsRef for SingleWriterTxKeyspace { + fn as_ref(&self) -> &Keyspace { + self.inner() + } +} + +impl SingleWriterTxKeyspace { + /// Returns the underlying LSM-tree's path. + #[must_use] + pub fn path(&self) -> PathBuf { + self.inner.path().into() + } + + /// Approximates the number of items in the keyspace. + /// + /// For update- or delete-heavy workloads, this value will + /// diverge from the real value, but is a O(1) operation. + /// + /// For insert-only workloads (e.g. logs, time series) + /// this value is reliable. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// assert_eq!(tree.approximate_len(), 0); + /// + /// tree.insert("1", "abc")?; + /// assert_eq!(tree.approximate_len(), 1); + /// + /// tree.remove("1")?; + /// // Oops! approximate_len will not be reliable here + /// assert_eq!(tree.approximate_len(), 2); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn approximate_len(&self) -> usize { + self.inner.approximate_len() + } + + /// Removes an item and returns its value if it existed. + /// + /// The operation will run wrapped in a transaction. + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let taken = tree.take("a")?.unwrap(); + /// assert_eq!(b"abc", &*taken); + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn take>(&self, key: K) -> crate::Result> { + self.fetch_update(key, |_| None) + } + + /// Atomically updates an item and returns the previous value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, Slice, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let prev = tree.fetch_update("a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let prev = tree.fetch_update("a", |_| None)?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn fetch_update, F: FnOnce(Option<&UserValue>) -> Option>( + &self, + key: K, + f: F, + ) -> crate::Result> { + let key: UserKey = key.into(); + + let mut tx = self.db.write_tx(); + + let prev = tx.fetch_update(self, key, f)?; + tx.commit()?; + + Ok(prev) + } + + /// Atomically updates an item and returns the new value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, Slice, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let updated = tree.update_fetch("a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"def", &*updated); + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let updated = tree.update_fetch("a", |_| None)?; + /// assert!(updated.is_none()); + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn update_fetch, F: FnOnce(Option<&UserValue>) -> Option>( + &self, + key: K, + f: F, + ) -> crate::Result> { + let key = key.into(); + + let mut tx = self.db.write_tx(); + let updated = tx.update_fetch(self, key, f)?; + tx.commit()?; + + Ok(updated) + } + + /// Inserts a key-value pair into the keyspace. + /// + /// Keys may be up to 65536 bytes long, values up to 2^32 bytes. + /// Shorter keys and values result in better performance. + /// + /// If the key already exists, the item will be overwritten. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// assert!(!db.read_tx().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn insert, V: Into>( + &self, + key: K, + value: V, + ) -> crate::Result<()> { + let mut tx = self.db.write_tx(); + tx.insert(self, key, value); + tx.commit()?; + Ok(()) + } + + /// Removes an item from the keyspace. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// assert!(!db.read_tx().is_empty(&tree)?); + /// + /// tree.remove("a")?; + /// assert!(db.read_tx().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn remove>(&self, key: K) -> crate::Result<()> { + let mut tx = self.db.write_tx(); + tx.remove(self, key); + tx.commit()?; + Ok(()) + } + + /// Removes an item from the keyspace, leaving behind a weak tombstone. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// The operation will run wrapped in a transaction. + /// + /// # Experimental + /// + /// This function is currently experimental. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// assert!(!db.read_tx().is_empty(&tree)?); + /// + /// tree.remove_weak("a")?; + /// assert!(db.read_tx().is_empty(&tree)?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn remove_weak>(&self, key: K) -> crate::Result<()> { + let mut tx = self.db.write_tx(); + tx.remove_weak(self, key); + tx.commit()?; + Ok(()) + } + + /// Retrieves an item from the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("my_value".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn get>(&self, key: K) -> crate::Result> { + self.inner.get(key) + } + + /// Retrieves the size of an item from the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// let len = tree.size_of("a")?.unwrap_or_default(); + /// assert_eq!("my_value".len() as u32, len); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn size_of>(&self, key: K) -> crate::Result> { + self.inner.size_of(key) + } + + /// Returns the first key-value pair in the keyspace. + /// The key in this pair is the minimum key in the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// tree.insert("b", "my_value")?; + /// + /// assert_eq!(b"a", &*tree.first_key_value().unwrap().key()?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn first_key_value(&self) -> Option { + let read_tx = self.db.read_tx(); + read_tx.first_key_value(self) + } + + /// Returns the last key-value pair in the keyspace. + /// The key in this pair is the maximum key in the keyspace. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// tree.insert("b", "my_value")?; + /// + /// assert_eq!(b"b", &*tree.last_key_value().unwrap().key()?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[must_use] + pub fn last_key_value(&self) -> Option { + let read_tx = self.db.read_tx(); + read_tx.last_key_value(self) + } + + /// Returns `true` if the keyspace contains the specified key. + /// + /// The operation will run wrapped in a read snapshot. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "my_value")?; + /// + /// assert!(tree.contains_key("a")?); + /// assert!(!tree.contains_key("b")?); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn contains_key>(&self, key: K) -> crate::Result { + self.inner.contains_key(key) + } + + /// Allows access to the inner keyspace handle, allowing to + /// escape from the transactional context. + #[doc(hidden)] + #[must_use] + pub fn inner(&self) -> &Keyspace { + &self.inner + } +} diff --git a/crates/fjall/src/tx/single_writer/mod.rs b/crates/fjall/src/tx/single_writer/mod.rs new file mode 100644 index 000000000..4aa57234f --- /dev/null +++ b/crates/fjall/src/tx/single_writer/mod.rs @@ -0,0 +1,189 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod keyspace; +mod write_tx; + +use crate::{ + keyspace::KeyspaceKey, Config, Database, KeyspaceCreateOptions, PersistMode, Snapshot, +}; +use std::{ + path::Path, + sync::{Arc, Mutex}, +}; + +pub use keyspace::SingleWriterTxKeyspace; +pub use write_tx::WriteTransaction; + +pub trait Openable { + fn open(config: Config) -> crate::Result + where + Self: Sized; +} + +/// Single-writer transactional database +#[derive(Clone)] +pub struct TxDatabase { + pub(crate) inner: Database, + single_writer_lock: Arc>, +} + +impl Openable for TxDatabase { + fn open(config: Config) -> crate::Result + where + Self: Sized, + { + Ok(Self { + inner: crate::Database::open(config)?, + single_writer_lock: Arc::default(), + }) + } +} + +impl TxDatabase { + /// Creates a new database builder to create or open a database at `path`. + pub fn builder(path: impl AsRef) -> crate::DatabaseBuilder { + crate::DatabaseBuilder::new(path.as_ref()) + } + + #[doc(hidden)] + #[must_use] + pub fn inner(&self) -> &Database { + &self.inner + } + + /// Starts a new writeable transaction. + #[must_use] + #[expect(clippy::missing_panics_doc)] + pub fn write_tx(&self) -> WriteTransaction<'_> { + #[expect(clippy::expect_used)] + let guard = self.single_writer_lock.lock().expect("poisoned tx lock"); + + let mut write_tx = WriteTransaction::new( + self.clone(), + self.inner.supervisor.snapshot_tracker.open(), + guard, + ); + + if !self.inner.config.manual_journal_persist { + write_tx = write_tx.durability(Some(PersistMode::Buffer)); + } + + write_tx + } + + /// Starts a new read-only transaction (a.k.a. [`Snapshot`]). + #[must_use] + pub fn read_tx(&self) -> Snapshot { + self.inner.snapshot() + } + + /// Flushes the active journal. The durability depends on the [`PersistMode`] + /// used. + /// + /// Persisting only affects durability, NOT consistency! Even without flushing + /// data is crash-safe. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{PersistMode, SingleWriterTxDatabase, KeyspaceCreateOptions}; + /// # let folder = tempfile::tempdir()?; + /// let db = SingleWriterTxDatabase::builder(folder).open()?; + /// let items = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + /// + /// items.insert("a", "hello")?; + /// + /// db.persist(PersistMode::SyncAll)?; + /// # + /// # Ok::<_, fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn persist(&self, mode: PersistMode) -> crate::Result<()> { + self.inner.persist(mode) + } + + /// Creates or opens a keyspace. + /// + /// If the keyspace does not yet exist, it will be created configured with `create_options`. + /// Otherwise simply a handle to the existing keyspace will be returned. + /// + /// Keyspace names can be up to 255 characters long and can not be empty. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + /// + /// # Panics + /// + /// Panics if the keyspace name is invalid. + pub fn keyspace( + &self, + name: &str, + create_options: impl FnOnce() -> KeyspaceCreateOptions, + ) -> crate::Result { + let keyspace = self.inner.keyspace(name, create_options)?; + + Ok(SingleWriterTxKeyspace { + inner: keyspace, + db: self.clone(), + }) + } + + /// Returns the number of keyspaces. + #[must_use] + pub fn keyspace_count(&self) -> usize { + self.inner.keyspace_count() + } + + /// Gets a list of all keyspace names in the database. + #[must_use] + pub fn list_keyspace_names(&self) -> Vec { + self.inner.list_keyspace_names() + } + + /// Returns `true` if the keyspace with the given name exists. + #[must_use] + pub fn keyspace_exists(&self, name: &str) -> bool { + self.inner.keyspace_exists(name) + } + + /// Returns the current write buffer size (active + sealed memtables). + #[must_use] + pub fn write_buffer_size(&self) -> u64 { + self.inner.write_buffer_size() + } + + /// Returns the number of journal fragments on disk. + #[must_use] + pub fn journal_count(&self) -> usize { + self.inner.journal_count() + } + + /// Returns the disk space usage of the entire database. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn disk_space(&self) -> crate::Result { + self.inner.disk_space() + } + + /// Opens a database in the given directory. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub fn open(config: Config) -> crate::Result { + let inner = Database::create_or_recover(config)?; + + Ok(Self { + inner, + single_writer_lock: Arc::default(), + }) + } +} diff --git a/crates/fjall/src/tx/single_writer/write_tx.rs b/crates/fjall/src/tx/single_writer/write_tx.rs new file mode 100644 index 000000000..8e702cc65 --- /dev/null +++ b/crates/fjall/src/tx/single_writer/write_tx.rs @@ -0,0 +1,375 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + snapshot_nonce::SnapshotNonce, + tx::{single_writer::keyspace::SingleWriterTxKeyspace, write_tx::BaseTransaction}, + Guard, Iter, Keyspace, PersistMode, Readable, SingleWriterTxDatabase, +}; +use lsm_tree::{UserKey, UserValue}; +use std::{ops::RangeBounds, sync::MutexGuard}; + +/// A single-writer (serialized) cross-keyspace transaction +/// +/// Use [`WriteTransaction::commit`] to commit changes to the keyspace(s). +/// +/// Transactions keep a consistent view of the database at the time, +/// meaning old data will not be dropped until it is not referenced by any active transaction. +/// +/// For that reason, you should try to keep transactions short-lived, and make sure they +/// are not held somewhere forever. +#[clippy::has_significant_drop] +pub struct WriteTransaction<'a> { + _guard: MutexGuard<'a, ()>, + inner: BaseTransaction, +} + +impl Readable for WriteTransaction<'_> { + fn get>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + self.inner.get(keyspace, key) + } + + fn contains_key>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result { + self.inner.contains_key(keyspace, key) + } + + fn first_key_value(&self, keyspace: impl AsRef) -> Option { + self.inner.first_key_value(keyspace) + } + + fn last_key_value(&self, keyspace: impl AsRef) -> Option { + self.inner.last_key_value(keyspace) + } + + fn size_of>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + self.inner.size_of(keyspace, key) + } + + fn iter(&self, keyspace: impl AsRef) -> Iter { + self.inner.iter(keyspace) + } + + fn range, R: RangeBounds>( + &self, + keyspace: impl AsRef, + range: R, + ) -> Iter { + self.inner.range(keyspace, range) + } + + fn prefix>(&self, keyspace: impl AsRef, prefix: K) -> Iter { + self.inner.prefix(keyspace, prefix) + } +} + +impl<'tx> WriteTransaction<'tx> { + pub(crate) fn new( + db: SingleWriterTxDatabase, + nonce: SnapshotNonce, + guard: MutexGuard<'tx, ()>, + ) -> Self { + Self { + _guard: guard, + inner: BaseTransaction::new(db.inner, nonce), + } + } + + /// Sets the durability level. + #[must_use] + pub fn durability(mut self, mode: Option) -> Self { + self.inner = self.inner.durability(mode); + self + } + + /// Removes an item and returns its value if it existed. + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx(); + /// + /// let taken = tx.take(&tree, "a")?.unwrap(); + /// assert_eq!(b"abc", &*taken); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn take>( + &mut self, + keyspace: &SingleWriterTxKeyspace, + key: K, + ) -> crate::Result> { + self.inner.take(keyspace.inner(), key) + } + + /// Atomically updates an item and returns the new value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Slice, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx(); + /// + /// let updated = tx.update_fetch(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"def", &*updated); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx(); + /// + /// let updated = tx.update_fetch(&tree, "a", |_| None)?; + /// assert!(updated.is_none()); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn update_fetch, F: FnOnce(Option<&UserValue>) -> Option>( + &mut self, + keyspace: &SingleWriterTxKeyspace, + key: K, + f: F, + ) -> crate::Result> { + self.inner.update_fetch(keyspace.inner(), key, f) + } + + /// Atomically updates an item and returns the previous value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Slice, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx(); + /// + /// let prev = tx.fetch_update(&tree, "a", |_| Some(Slice::from(*b"def")))?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert_eq!(Some("def".as_bytes().into()), item); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # use std::sync::Arc; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "abc")?; + /// + /// let mut tx = db.write_tx(); + /// + /// let prev = tx.fetch_update(&tree, "a", |_| None)?.unwrap(); + /// assert_eq!(b"abc", &*prev); + /// tx.commit()?; + /// + /// let item = tree.get("a")?; + /// assert!(item.is_none()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn fetch_update, F: FnOnce(Option<&UserValue>) -> Option>( + &mut self, + keyspace: &SingleWriterTxKeyspace, + key: K, + f: F, + ) -> crate::Result> { + self.inner.fetch_update(keyspace.inner(), key, f) + } + + // Inserts a key-value pair into the keyspace. + /// + /// Keys may be up to 65536 bytes long, values up to 2^32 bytes. + /// Shorter keys and values result in better performance. + /// + /// If the key already exists, the item will be overwritten. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "previous_value")?; + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// + /// let mut tx = db.write_tx(); + /// tx.insert(&tree, "a", "new_value"); + /// + /// drop(tx); + /// + /// // Write was not committed + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn insert, V: Into>( + &mut self, + keyspace: &SingleWriterTxKeyspace, + key: K, + value: V, + ) { + self.inner.insert(keyspace.inner(), key, value); + } + + /// Removes an item from the keyspace. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "previous_value")?; + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// + /// let mut tx = db.write_tx(); + /// tx.remove(&tree, "a"); + /// + /// // Read-your-own-write + /// let item = tx.get(&tree, "a")?; + /// assert_eq!(None, item); + /// + /// drop(tx); + /// + /// // Deletion was not committed + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + pub fn remove>(&mut self, keyspace: &SingleWriterTxKeyspace, key: K) { + self.inner.remove(keyspace.inner(), key); + } + + /// Removes an item from the keyspace, leaving behind a weak tombstone. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// # Experimental + /// + /// This function is currently experimental. + /// + /// # Examples + /// + /// ``` + /// # use fjall::{SingleWriterTxDatabase, KeyspaceCreateOptions, Readable}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let db = SingleWriterTxDatabase::builder(folder).open()?; + /// # let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + /// tree.insert("a", "previous_value")?; + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// + /// let mut tx = db.write_tx(); + /// tx.remove_weak(&tree, "a"); + /// + /// // Read-your-own-write + /// let item = tx.get(&tree, "a")?; + /// assert_eq!(None, item); + /// + /// drop(tx); + /// + /// // Deletion was not committed + /// assert_eq!(b"previous_value", &*tree.get("a")?.unwrap()); + /// # + /// # Ok::<(), fjall::Error>(()) + /// ``` + #[doc(hidden)] + pub fn remove_weak>(&mut self, keyspace: &SingleWriterTxKeyspace, key: K) { + self.inner.remove_weak(keyspace.inner(), key); + } + + /// Commits the transaction. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn commit(self) -> crate::Result<()> { + self.inner.commit() + } + + /// More explicit alternative to dropping the transaction + /// to roll it back. + pub fn rollback(self) { + self.inner.rollback(); + } +} diff --git a/crates/fjall/src/tx/write_tx.rs b/crates/fjall/src/tx/write_tx.rs new file mode 100644 index 000000000..e339989e9 --- /dev/null +++ b/crates/fjall/src/tx/write_tx.rs @@ -0,0 +1,404 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + batch::item::Item, snapshot_nonce::SnapshotNonce, Database, Guard, HashMap, Iter, Keyspace, + OwnedWriteBatch, PersistMode, Readable, +}; +use lsm_tree::{AbstractTree, InternalValue, Memtable, SeqNo, UserKey, UserValue}; +use std::{ops::RangeBounds, sync::Arc}; + +fn ignore_tombstone_value(item: InternalValue) -> Option { + if item.is_tombstone() { + None + } else { + Some(item) + } +} + +pub(super) struct BaseTransaction { + /// Database to work with + pub(super) db: Database, + + /// Ephemeral transaction changes + /// + /// Used for RYOW (read-your-own-writes) + pub(crate) memtables: HashMap>, + + /// The snapshot, for repeatable reads + pub(crate) nonce: SnapshotNonce, + + /// Durability level used, see [`PersistMode`]. + pub(crate) durability: Option, + + /// Current sequence number + /// + /// The sequence number starts at 0b1000...0000 and is incremented with each write. + /// + /// This ensures that writes within the transaction are always newer than any existing data. + pub(crate) seqno: SeqNo, +} + +impl Readable for BaseTransaction { + fn get>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + let keyspace = keyspace.as_ref(); + let key = key.as_ref(); + + if let Some(memtable) = self.memtables.get(keyspace) { + if let Some(item) = memtable.get(key, SeqNo::MAX) { + return Ok(ignore_tombstone_value(item).map(|x| x.value)); + } + } + + let res = keyspace.tree.get(key, self.nonce.instant)?; + + Ok(res) + } + + fn contains_key>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result { + let keyspace = keyspace.as_ref(); + let key = key.as_ref(); + + if let Some(memtable) = self.memtables.get(keyspace) { + if let Some(item) = memtable.get(key, SeqNo::MAX) { + return Ok(!item.key.is_tombstone()); + } + } + + let contains = keyspace.tree.contains_key(key, self.nonce.instant)?; + + Ok(contains) + } + + fn first_key_value(&self, keyspace: impl AsRef) -> Option { + self.iter(keyspace.as_ref()).next() + } + + fn last_key_value(&self, keyspace: impl AsRef) -> Option { + self.iter(keyspace.as_ref()).next_back() + } + + fn size_of>( + &self, + keyspace: impl AsRef, + key: K, + ) -> crate::Result> { + let keyspace = keyspace.as_ref(); + let key = key.as_ref(); + + if let Some(memtable) = self.memtables.get(keyspace) { + if let Some(item) = memtable.get(key, SeqNo::MAX) { + // NOTE: Values are limited to u32 in lsm-tree + #[expect(clippy::cast_possible_truncation)] + return Ok(ignore_tombstone_value(item).map(|x| x.value.len() as u32)); + } + } + + let res = keyspace.tree.size_of(key, self.nonce.instant)?; + + Ok(res) + } + + fn iter(&self, keyspace: impl AsRef) -> Iter { + let keyspace = keyspace.as_ref(); + + let iter = keyspace.tree.iter( + self.nonce.instant, + self.memtables + .get(keyspace) + .cloned() + .map(|mt| (mt, self.seqno)), + ); + + Iter::new(self.nonce.clone(), iter) + } + + fn range, R: RangeBounds>( + &self, + keyspace: impl AsRef, + range: R, + ) -> Iter { + let keyspace = keyspace.as_ref(); + + let iter = keyspace.tree.range( + range, + self.nonce.instant, + self.memtables + .get(keyspace) + .cloned() + .map(|mt| (mt, self.seqno)), + ); + + Iter::new(self.nonce.clone(), iter) + } + + fn prefix>(&self, keyspace: impl AsRef, prefix: K) -> Iter { + let keyspace = keyspace.as_ref(); + + let iter = keyspace.tree.prefix( + prefix, + self.nonce.instant, + self.memtables + .get(keyspace) + .cloned() + .map(|mt| (mt, self.seqno)), + ); + + Iter::new(self.nonce.clone(), iter) + } +} + +impl BaseTransaction { + pub(crate) fn new(db: Database, nonce: SnapshotNonce) -> Self { + Self { + db, + memtables: HashMap::default(), + nonce, + durability: None, + seqno: 0x8000_0000_0000_0000, + } + } + + /// Sets the durability level. + #[must_use] + pub(super) fn durability(mut self, mode: Option) -> Self { + self.durability = mode; + self + } + + /// Removes an item and returns its value if it existed. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub(super) fn take>( + &mut self, + keyspace: &Keyspace, + key: K, + ) -> crate::Result> { + self.fetch_update(keyspace, key, |_| None) + } + + /// Atomically updates an item and returns the new value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub(super) fn update_fetch< + K: Into, + F: FnOnce(Option<&UserValue>) -> Option, + >( + &mut self, + keyspace: &Keyspace, + key: K, + f: F, + ) -> crate::Result> { + let key = key.into(); + let prev = self.get(keyspace, &key)?; + let updated = f(prev.as_ref()); + + if let Some(value) = updated.clone() { + // NOTE: Skip insert if the value hasn't changed + if prev.as_ref() != Some(&value) { + self.insert(keyspace, key, value); + } + } else if prev.is_some() { + self.remove(keyspace, key); + } + + Ok(updated) + } + + /// Atomically updates an item and returns the previous value. + /// + /// Returning `None` removes the item if it existed before. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub(super) fn fetch_update< + K: Into, + F: FnOnce(Option<&UserValue>) -> Option, + >( + &mut self, + keyspace: &Keyspace, + key: K, + f: F, + ) -> crate::Result> { + let key = key.into(); + let prev = self.get(keyspace, &key)?; + let updated = f(prev.as_ref()); + + if let Some(value) = updated { + // NOTE: Skip insert if the value hasn't changed + if prev.as_ref() != Some(&value) { + self.insert(keyspace, key, value); + } + } else if prev.is_some() { + self.remove(keyspace, key); + } + + Ok(prev) + } + + /// Inserts a key-value pair into the keyspace. + /// + /// Keys may be up to 65536 bytes long, values up to 2^32 bytes. + /// Shorter keys and values result in better performance. + /// + /// If the key already exists, the item will be overwritten. + pub(super) fn insert, V: Into>( + &mut self, + keyspace: &Keyspace, + key: K, + value: V, + ) { + self.memtables + .entry(keyspace.clone()) + .or_insert_with(|| Arc::new(Memtable::new(0))) + .insert(lsm_tree::InternalValue::from_components( + key, + value, + self.seqno, + lsm_tree::ValueType::Value, + )); + + self.seqno += 1; + } + + /// Removes an item from the keyspace. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + pub(super) fn remove>(&mut self, keyspace: &Keyspace, key: K) { + self.memtables + .entry(keyspace.clone()) + .or_insert_with(|| Arc::new(Memtable::new(0))) + .insert(lsm_tree::InternalValue::new_tombstone(key, self.seqno)); + + self.seqno += 1; + } + + /// Removes an item from the keyspace, leaving behind a weak tombstone. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// The key may be up to 65536 bytes long. + /// Shorter keys result in better performance. + pub(super) fn remove_weak>(&mut self, keyspace: &Keyspace, key: K) { + self.memtables + .entry(keyspace.clone()) + .or_insert_with(|| Arc::new(Memtable::new(0))) + .insert(lsm_tree::InternalValue::new_weak_tombstone(key, self.seqno)); + + self.seqno += 1; + } + + /// Commits the transaction. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub(super) fn commit(self) -> crate::Result<()> { + // skip all the logic if no keys were written to + if self.memtables.is_empty() { + return Ok(()); + } + + // TODO: instead of using batch, write batch::commit as a generic function that takes + // a impl Iterator + // that way, we don't have to move the memtable(s) into the batch first to commit + let mut batch = OwnedWriteBatch::new(self.db).durability(self.durability); + + for (keyspace, memtable) in self.memtables { + let mut prev_key: Option = None; + + for item in memtable.iter() { + if let Some(prev_key) = &prev_key { + if item.key.user_key == prev_key { + continue; + } + } + + batch.data.push(Item::new( + keyspace.clone(), + item.key.user_key.clone(), + item.value.clone(), + item.key.value_type, + )); + + prev_key = Some(item.key.user_key.clone()); + } + } + + batch.commit()?; + + Ok(()) + } + + /// More explicit alternative to dropping the transaction + /// to roll it back. + #[expect(clippy::unused_self)] + pub(super) fn rollback(self) {} +} + +#[cfg(test)] +mod tests { + use crate::{ + tx::{single_writer::SingleWriterTxKeyspace, write_tx::BaseTransaction}, + KeyspaceCreateOptions, SingleWriterTxDatabase, + }; + use tempfile::TempDir; + + struct TestEnv { + db: SingleWriterTxDatabase, + tree: SingleWriterTxKeyspace, + + #[expect(unused)] + tmpdir: TempDir, + } + + fn setup() -> Result> { + let tmpdir: TempDir = tempfile::tempdir()?; + let db = SingleWriterTxDatabase::builder(tmpdir.path()).open()?; + + let tree = db.keyspace("foo", KeyspaceCreateOptions::default)?; + + Ok(TestEnv { db, tree, tmpdir }) + } + + #[test] + fn update_fetch() -> Result<(), Box> { + let env = setup()?; + + env.tree.insert([2u8], [20u8])?; + + let mut tx = BaseTransaction::new( + env.db.inner.clone(), + env.db.inner().supervisor.snapshot_tracker.open(), + ); + + let new = tx.update_fetch(env.tree.inner(), [2u8], |v| { + v.and_then(|v| v.first().copied()).map(|v| [v + 5].into()) + })?; + + assert_eq!(new, Some([25u8].into())); + tx.commit()?; + + Ok(()) + } +} diff --git a/crates/fjall/src/version.rs b/crates/fjall/src/version.rs new file mode 100644 index 000000000..ac2113b03 --- /dev/null +++ b/crates/fjall/src/version.rs @@ -0,0 +1,151 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use byteorder::WriteBytesExt; + +/// Disk format version +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum FormatVersion { + /// Version for 1.x.x releases + V1, + + /// Version for 2.x.x releases + V2, + + /// Version for 3.x.x releases + V3, + + /// Fixed-width key/value lengths in LSM table data blocks + V4, +} + +impl std::fmt::Display for FormatVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", u8::from(*self)) + } +} + +impl From for u8 { + fn from(value: FormatVersion) -> Self { + match value { + FormatVersion::V1 => 1, + FormatVersion::V2 => 2, + FormatVersion::V3 => 3, + FormatVersion::V4 => 4, + } + } +} + +impl TryFrom for FormatVersion { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + 1 => Ok(Self::V1), + 2 => Ok(Self::V2), + 3 => Ok(Self::V3), + 4 => Ok(Self::V4), + _ => Err(()), + } + } +} + +const MAGIC_BYTES: [u8; 3] = *b"FJL"; + +impl FormatVersion { + pub(crate) fn parse_file_header(bytes: &[u8]) -> Option { + let first_three = bytes.get(0..3)?; + + if first_three == MAGIC_BYTES { + let version = *bytes.get(3)?; + let version = Self::try_from(version).ok()?; + Some(version) + } else { + None + } + } + + pub(crate) fn write_file_header( + self, + writer: &mut W, + ) -> std::io::Result<()> { + writer.write_all(&MAGIC_BYTES)?; + writer.write_u8(u8::from(self))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + pub fn version_serialize() -> crate::Result<()> { + let mut bytes = vec![]; + FormatVersion::V1.write_file_header(&mut bytes)?; + assert_eq!(bytes, &[b'F', b'J', b'L', 1]); + Ok(()) + } + + #[test] + pub fn version_serialize_2() -> crate::Result<()> { + let mut bytes = vec![]; + FormatVersion::V2.write_file_header(&mut bytes)?; + assert_eq!(bytes, &[b'F', b'J', b'L', 2]); + Ok(()) + } + + #[test] + pub fn version_deserialize_success() { + let version = FormatVersion::parse_file_header(&[b'F', b'J', b'L', 1]); + assert_eq!(version, Some(FormatVersion::V1)); + } + + #[test] + pub fn version_deserialize_success_2() { + let version = FormatVersion::parse_file_header(&[b'F', b'J', b'L', 2]); + assert_eq!(version, Some(FormatVersion::V2)); + } + + #[test] + pub fn version_serialize_and_deserialize_4() -> crate::Result<()> { + let mut bytes = vec![]; + FormatVersion::V4.write_file_header(&mut bytes)?; + assert_eq!(bytes, &[b'F', b'J', b'L', 4]); + assert_eq!( + FormatVersion::parse_file_header(&bytes), + Some(FormatVersion::V4), + ); + Ok(()) + } + + #[test] + pub fn version_deserialize_fail() { + let version = FormatVersion::parse_file_header(&[b'F', b'J', b'X', 1]); + assert!(version.is_none()); + } + + #[test] + #[expect(clippy::expect_used)] + pub fn version_serde_round_trip() { + let mut buf = vec![]; + FormatVersion::V1 + .write_file_header(&mut buf) + .expect("can't fail"); + + let version = FormatVersion::parse_file_header(&buf); + assert_eq!(version, Some(FormatVersion::V1)); + } + + #[test] + #[expect(clippy::expect_used)] + pub fn version_len() { + let mut buf = vec![]; + FormatVersion::V1 + .write_file_header(&mut buf) + .expect("can't fail"); + assert_eq!(4, buf.len()); + } +} diff --git a/crates/fjall/src/worker_pool.rs b/crates/fjall/src/worker_pool.rs new file mode 100644 index 000000000..a8eecb220 --- /dev/null +++ b/crates/fjall/src/worker_pool.rs @@ -0,0 +1,270 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + compaction::worker::run as run_compaction, flush::worker::run as run_flush, poison::PoisonDart, + stats::Stats, supervisor::Supervisor, Keyspace, +}; +use lsm_tree::MemtableId; +use std::{ + borrow::Cow, + sync::{atomic::AtomicUsize, Arc, Mutex}, + thread::JoinHandle, +}; + +pub enum WorkerMessage { + Flush, + Compact(Keyspace), + Close, + RotateMemtable(Keyspace, MemtableId), +} + +impl std::fmt::Debug for WorkerMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Flush => Cow::Borrowed("WorkerMessage:Flush"), + Self::Compact(k) => Cow::Owned(format!("WorkerMessage:Compact({:?})", k.name)), + Self::Close => Cow::Borrowed("WorkerMessage:Close"), + Self::RotateMemtable(k, memtable_id) => + Cow::Owned(format!("WorkerMessage:Rotate({:?}, {memtable_id})", k.name)), + } + ) + } +} + +type WorkerHandle = JoinHandle>; + +pub struct WorkerPool { + thread_handles: Mutex>, + pub(crate) rx: flume::Receiver, + pub(crate) sender: flume::Sender, +} + +impl WorkerPool { + pub fn prepare() -> Self { + let (sender, rx) = flume::bounded(1_000); + + Self { + thread_handles: Mutex::default(), + rx, + sender, + } + } + + pub fn start( + &self, + pool_size: usize, + supervisor: &Supervisor, + stats: &Arc, + poison_dart: &PoisonDart, + thread_counter: &Arc, + ) -> crate::Result<()> { + use std::sync::atomic::Ordering::Relaxed; + + log::debug!("Starting worker pool with {pool_size} threads"); + + thread_counter.fetch_add(pool_size, Relaxed); + + let thread_handles = (0..pool_size) + .map(|i| { + std::thread::Builder::new() + .name("fjall:worker".to_string()) + .spawn({ + log::trace!("Starting fjall worker thread #{i}"); + + let worker_state = WorkerState { + pool_size, + worker_id: i, + rx: self.rx.clone(), + supervisor: supervisor.clone(), + stats: stats.clone(), + sender: self.sender.clone(), + }; + + let thread_counter = thread_counter.clone(); + let poison_dart = poison_dart.clone(); + + move || loop { + match worker_tick(&worker_state) { + Ok(should_abort) => { + if should_abort { + log::debug!("Worker #{i} closes because DB is dropping"); + thread_counter.fetch_sub(1, Relaxed); + return Ok(()); + } + } + Err(e) => { + log::error!("Worker #{i} crashed: {e:?}"); + poison_dart.poison(); + return Err(e); + } + } + } + }) + .inspect_err(|_| { + thread_counter.fetch_sub(1, Relaxed); + }) + }) + .collect::>()?; + + *self.thread_handles.lock().expect("lock is poisoned") = thread_handles; + + Ok(()) + } +} + +struct WorkerState { + pool_size: usize, + worker_id: usize, + supervisor: Supervisor, + rx: flume::Receiver, + sender: flume::Sender, + stats: Arc, +} + +fn worker_tick(ctx: &WorkerState) -> crate::Result { + let Ok(item) = ctx.rx.recv() else { + return Ok(true); + }; + + log::trace!("Worker #{} got message: {item:?}", ctx.worker_id); + + match item { + WorkerMessage::Close => { + return Ok(true); + } + WorkerMessage::RotateMemtable(keyspace, memtable_id) => { + log::trace!("acquiring journal lock"); + let journal_writer = keyspace.supervisor.journal.get_writer()?; + keyspace.inner_rotate_memtable(journal_writer, memtable_id)?; + } + WorkerMessage::Flush => { + let Some(task) = ctx.supervisor.flush_manager.dequeue() else { + return Ok(false); + }; + + { + log::trace!("acquiring journal lock to maybe rotate journal"); + let mut journal_writer = ctx.supervisor.journal.get_writer()?; + + if journal_writer.pos()? > 64_000_000 { + #[expect(clippy::expect_used)] + let mut journal_manager = ctx + .supervisor + .journal_manager + .write() + .expect("lock is poisoned"); + + let seqno_map = { + #[expect(clippy::expect_used)] + let keyspaces = ctx.supervisor.keyspaces.write().expect("lock is poisoned"); + + crate::supervisor::Supervisor::build_seqno_map(&keyspaces) + }; + + journal_manager.rotate_journal(&mut journal_writer, seqno_map)?; + + if journal_manager.disk_space_used() + >= ctx.supervisor.db_config.max_journaling_size_in_bytes + { + let stragglers = + journal_manager.get_keyspaces_to_flush_for_oldest_journal_eviction(); + + for keyspace in stragglers { + log::info!( + "Rotating {:?} to try to reduce journal size", + keyspace.name, + ); + keyspace.request_rotation(); + } + } + } + } + + run_flush( + &task, + &ctx.supervisor.write_buffer_size, + &ctx.supervisor.snapshot_tracker, + &ctx.stats, + )?; + + for _ in 0..ctx.pool_size { + ctx.sender + .try_send(WorkerMessage::Compact(task.keyspace.clone())) + .ok(); + } + + ctx.supervisor + .journal_manager + .write() + .expect("lock is poisoned") + .maintenance()?; + } + WorkerMessage::Compact(keyspace) => { + // NOTE: Let one worker prioritize flushing if there are pending flushes + // + // Disable when only 1 worker exists to avoid deadlock + if ctx.pool_size > 1 && ctx.worker_id == 0 { + ctx.sender.send(WorkerMessage::Compact(keyspace)).ok(); + return Ok(false); + } + + run_compaction(&keyspace, &ctx.supervisor.snapshot_tracker, &ctx.stats)?; + } + } + + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AbstractTree, Database, KeyspaceCreateOptions}; + use test_log::test; + + // https://github.com/fjall-rs/fjall/pull/303 + #[test] + fn keyspace_compact_after_startup() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let ks = db.keyspace("default", KeyspaceCreateOptions::default)?; + + ks.insert("a", "a")?; + ks.rotate_memtable_and_wait()?; + + ks.insert("a", "a")?; + ks.rotate_memtable_and_wait()?; + + ks.insert("a", "a")?; + ks.rotate_memtable_and_wait()?; + + assert!(ks.tree.l0_run_count() > 0); + } + + { + let db = Database::builder(&folder) + .worker_threads_unchecked(0) + .open()?; + + assert_eq!( + 1, + db.worker_pool.rx.len(), + "worker message should be enqueued on startup", + ); + let item = db.worker_pool.rx.try_recv().expect("should get message"); + assert!( + matches!(item, WorkerMessage::Compact(_)), + "worker message should be compaction request", + ); + } + + Ok(()) + } +} diff --git a/crates/fjall/src/write_buffer_manager.rs b/crates/fjall/src/write_buffer_manager.rs new file mode 100644 index 000000000..d06d9065a --- /dev/null +++ b/crates/fjall/src/write_buffer_manager.rs @@ -0,0 +1,76 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::sync::{atomic::AtomicU64, Arc}; + +/// Keeps track of the size of the database's write buffer +#[derive(Clone, Default, Debug)] +pub struct WriteBufferManager(Arc); + +impl std::ops::Deref for WriteBufferManager { + type Target = AtomicU64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl WriteBufferManager { + pub fn get(&self) -> u64 { + self.load(std::sync::atomic::Ordering::Acquire) + } + + // Adds some bytes to the write buffer counter. + // + // Returns the counter *after* incrementing. + pub fn allocate(&self, n: u64) -> u64 { + let before = self.fetch_add(n, std::sync::atomic::Ordering::AcqRel); + before + n + } + + // Frees some bytes from the write buffer counter. + // + // Returns the counter *after* decrementing. + pub fn free(&self, n: u64) -> u64 { + use std::sync::atomic::Ordering::{Acquire, SeqCst}; + + loop { + let now = self.load(Acquire); + let subbed = now.saturating_sub(n); + + if self.compare_exchange(now, subbed, SeqCst, SeqCst).is_ok() { + return subbed; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn write_buffer_manager_increment() { + let m = WriteBufferManager::default(); + m.allocate(5); + assert_eq!(m.get(), 5); + + m.allocate(15); + assert_eq!(m.get(), 20); + } + + #[test] + fn write_buffer_manager_decrement() { + let m = WriteBufferManager::default(); + m.allocate(20); + assert_eq!(m.get(), 20); + + m.free(5); + assert_eq!(m.get(), 15); + + m.free(20); + assert_eq!(m.get(), 0); + } +} diff --git a/crates/fjall/test_fixture/v1_keyspace/partitions/a/config b/crates/fjall/test_fixture/v1_keyspace/partitions/a/config new file mode 100644 index 0000000000000000000000000000000000000000..61e5bd120bd4eff9fc05bda9d16a304c1be34a9d GIT binary patch literal 16 XcmZ?s^6_zYb2nsQWME(rU|c$=tIj675%n){E;HYq&v)Ov;hC$I%1Y&Ci9Oi4 z^a%RF%oYQL{6&kQNK%7OKaSoO}Sr=2#Iprx;$F_=4`lwxjqAkLC^p&Plcp&{5xCUtwyICV9YVD_51rAblJ-2tTx#ObQ2_>`;Ykr*hTAYip4YHBu^NT?EaW~OBkj;X z1`Wfw3Ll0^ZjqcSOZz3=ky|?A9^KGW+@?unh&I=}#zV2R07ug!#Jj}Lxmq{EE5%|Q z#|4^OM>SEf)OIoE+H#kgi{gIuVr%>1`hrwwGpZxe-m^*y>vkTuU_umdFaq>ex_e66DbIF$UG_eu(!{?0ssDeK?0I9o GQu+m3qJv8S literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace/partitions/a/version b/crates/fjall/test_fixture/v1_keyspace/partitions/a/version new file mode 100644 index 0000000000000000000000000000000000000000..2073e5dd0ed44d0784b77c9a924a0958da4139c7 GIT binary patch literal 5 McmZ?s@?l^E00VFU-T(jq literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace/partitions/b/config b/crates/fjall/test_fixture/v1_keyspace/partitions/b/config new file mode 100644 index 0000000000000000000000000000000000000000..61e5bd120bd4eff9fc05bda9d16a304c1be34a9d GIT binary patch literal 16 XcmZ?s^6_zYb2nsQWME(rU|JLVOarxq29GkoUY0tzJqmGOmSq!y*B7c1oDE9B>;$}t3R17%Wd87BCbmMD~f zB;8;Z%Wt~f0n`EmqWnPC2M~`DCIr+8SIyaKG7m1!z<{Z`5~@1b$;}X^FaXLB44EqH zZtA~#O3YV~fx*5mFeM1P*d$hbdOGPWR0j(h4Koo&S3(&t85jgWOh&jtzz_oiZcxB6 cra*Zx^~<6Bt56z7{eUrs3k?eAkRTsJ09!&iy#N3J literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace/partitions/b/version b/crates/fjall/test_fixture/v1_keyspace/partitions/b/version new file mode 100644 index 0000000000000000000000000000000000000000..2073e5dd0ed44d0784b77c9a924a0958da4139c7 GIT binary patch literal 5 McmZ?s@?l^E00VFU-T(jq literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace/partitions/c/config b/crates/fjall/test_fixture/v1_keyspace/partitions/c/config new file mode 100644 index 0000000000000000000000000000000000000000..61e5bd120bd4eff9fc05bda9d16a304c1be34a9d GIT binary patch literal 16 XcmZ?s^6_zYb2nsQWME(rU|c$=tIj675%n){E;HYq&v)Ov;hC$I%1Y&Ci9Oi4 z^a%RF%oYQL{6&kQNK%7OKaSoO}Sr=2#Iprx;$F_=4`lwxjqAkLC^p&Plcp&{5xCUtwyICV9YVD_51rAblJ-2tTx#ObQ2_>`;Ykr*hTAYip4YHBu^NT?EaW~OBkj;X z1`Wfw3Ll0^ZjqcSOZz3=ky|?A9^KGW+@?unh&I=}#zV2R07ug!#Jj}Lxmq{EE5%|Q z#|4^OM>SEf)OIoE+H#kgi{gIuVr%>1`hrwwGpZxe-m^*y>vkTuU_umdFaq>ex_e66DbIF$UG_eu(!{?0ssDeK?0I9o GQu+m3qJv8S literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/version b/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/a/version new file mode 100644 index 0000000000000000000000000000000000000000..2073e5dd0ed44d0784b77c9a924a0958da4139c7 GIT binary patch literal 5 McmZ?s@?l^E00VFU-T(jq literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/config b/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/config new file mode 100644 index 0000000000000000000000000000000000000000..61e5bd120bd4eff9fc05bda9d16a304c1be34a9d GIT binary patch literal 16 XcmZ?s^6_zYb2nsQWME(rU|JLVOarxq29GkoUY0tzJqmGOmSq!y*B7c1oDE9B>;$}t3R17%Wd87BCbmMD~f zB;8;Z%Wt~f0n`EmqWnPC2M~`DCIr+8SIyaKG7m1!z<{Z`5~@1b$;}X^FaXLB44EqH zZtA~#O3YV~fx*5mFeM1P*d$hbdOGPWR0j(h4Koo&S3(&t85jgWOh&jtzz_oiZcxB6 cra*Zx^~<6Bt56z7{eUrs3k?eAkRTsJ09!&iy#N3J literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/version b/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/b/version new file mode 100644 index 0000000000000000000000000000000000000000..2073e5dd0ed44d0784b77c9a924a0958da4139c7 GIT binary patch literal 5 McmZ?s@?l^E00VFU-T(jq literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/config b/crates/fjall/test_fixture/v1_keyspace_corrupt_journal/partitions/c/config new file mode 100644 index 0000000000000000000000000000000000000000..61e5bd120bd4eff9fc05bda9d16a304c1be34a9d GIT binary patch literal 16 XcmZ?s^6_zYb2nsQWME(rU|WP%A{BgsfIl8huH$w)Gij3gt;NWcBRp`0iu z%87EKoG2&CiE^TxC@0E^zkfD|tSl?b%CfSoEGx^(va+l!E6d8V@~qGD(--sweL-K) z7xV>vL0`}p^aXuEU(gpcBh5%N(u_1C%}6uSj5MR^W)v^Q^_SbV%d7I~EsLeoTZ)rl z`Wu5kPl9oE@?0Eu!`+y5W|C*av*FnoogZc|*bDZ8y)b>UAQ{L&1~QO=gd`*(2}wvo z5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!HLK2dYgd`*(2}wvo5|WUFBqSjT zNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!HLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TX sBq0e&NJ0{lkc1>8Aqh!HLK2dYgd`*(3ICgfaU8upRySUEho`H40!(|$rvLx| literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace/partitions/default1/config b/crates/fjall/test_fixture/v2_keyspace/partitions/default1/config new file mode 100644 index 0000000000000000000000000000000000000000..1fe458313565a33761516e13c30a898c07a3c7d7 GIT binary patch literal 30 bcmZ?s@?m0UU}OLR0Wb!#xfobDSU@5G9L@nE literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace/partitions/default1/levels b/crates/fjall/test_fixture/v2_keyspace/partitions/default1/levels new file mode 100644 index 0000000000000000000000000000000000000000..23ecc55259e97ef90d339c668eb4de3ffd005aff GIT binary patch literal 41 TcmeYX_GMycU|?VbV!QwVCU^k% literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace/partitions/default1/manifest b/crates/fjall/test_fixture/v2_keyspace/partitions/default1/manifest new file mode 100644 index 0000000000000000000000000000000000000000..969bbb4ca73b53322008098eee075a57d6f70e73 GIT binary patch literal 7 OcmeYX_GMyVU}AMbL_{r8R4*}-guaDJIYhnNm=(#(_F-}}wM zVQDV2pfhtM`ow(1(^SGKHHmwSHC!?lcB)iA4U>F#Z%aJ2E13(=$FW0?qd1xAygWh;A5>&MMY*GxCoZ{C4904%rt1n{{~ zR~Y&oPoWd}qJtsFScY7Av?-xqfM&|+x&WptQ&FA_V}FNxZ*kOy#%OArM@|}aOSPd!I;MN-eSM6 zhGk$b6=kmZKr~iOuY{?`1uD;082b?|oj|5>33YQ~l!V0&0+hrG6~Wz_S|Gm)HncJ` nEyJuGGz`_6kEfxVS literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace/partitions/default2/levels b/crates/fjall/test_fixture/v2_keyspace/partitions/default2/levels new file mode 100644 index 0000000000000000000000000000000000000000..23ecc55259e97ef90d339c668eb4de3ffd005aff GIT binary patch literal 41 TcmeYX_GMycU|?VbV!QwVCU^k% literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace/partitions/default2/manifest b/crates/fjall/test_fixture/v2_keyspace/partitions/default2/manifest new file mode 100644 index 0000000000000000000000000000000000000000..445fd09222e5aa0f191025b0b5bba2d4bb9a6b3d GIT binary patch literal 7 OcmeYX_GMyZU2LN3~C?fy> literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace/version b/crates/fjall/test_fixture/v2_keyspace/version new file mode 100644 index 000000000..29e6c1aa3 --- /dev/null +++ b/crates/fjall/test_fixture/v2_keyspace/version @@ -0,0 +1 @@ +FJL \ No newline at end of file diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/journals/2 b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/journals/2 new file mode 100644 index 0000000000000000000000000000000000000000..908e21c951aa6d901390db0db101fdd79dd213c5 GIT binary patch literal 16777216 zcmeF)yGkoj6oBEK42YK}P*8Ru*hXt>5fMwpa?q^~#01Plu(A+rYy=yh!6y(4!BQ-2 z?0pp{iWtXBW9cWH<^)3cnECfwo}<$UAvCH#gRjvLnqhQ1?d1J-IUgFS`n##kZa;2y z_fyQ<#bJs?Czg9DE~mWAj@Q>Vo))ifmsVDrgZFrUpRwBe>{PGK998d=7oE8>ZXI-c zy`r-ldu3jx*e*J$mrdV|TwQKIpVi-MeE7W)>r>aWA+R+zal7_Z>zL^gs{vKo35?ucL{YsEL}WiJCb4{eI_ybHTabTyQQp7n}>9 z|1I3v=xlU0Ivbsh&PHdWv$6hv<;(}>1M`9Tzyu!-I{aK58Z>@-W|C{ewc*-$dw!UE!M)&Ka4&p)vY;|3 zgEAi;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?v_l61 z00000g8Z!qxezj7z<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r Q3>YwAz<>b*1`MnR0?}N{hX4Qo literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/config b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/config new file mode 100644 index 0000000000000000000000000000000000000000..1fe458313565a33761516e13c30a898c07a3c7d7 GIT binary patch literal 30 bcmZ?s@?m0UU}OLR0Wb!#xfobDSU@5G9L@nE literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/levels b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/levels new file mode 100644 index 0000000000000000000000000000000000000000..23ecc55259e97ef90d339c668eb4de3ffd005aff GIT binary patch literal 41 TcmeYX_GMycU|?VbV!QwVCU^k% literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/manifest b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default1/manifest new file mode 100644 index 0000000000000000000000000000000000000000..969bbb4ca73b53322008098eee075a57d6f70e73 GIT binary patch literal 7 OcmeYX_GMyVU}AMbL_{r8R4*}-guaDJIYhnNm=(#(_F-}}wM zVQDV2pfhtM`ow(1(^SGKHHmwSHC!?lcB)iA4U>F#Z%aJ2E13(=$FW0?qd1xAygWh;A5>&MMY*GxCoZ{C4904%rt1n{{~ zR~Y&oPoWd}qJtsFScY7Av?-xqfM&|+x&WptQ&FA_V}FNxZ*kOy#%OArM@|}aOSPd!I;MN-eSM6 zhGk$b6=kmZKr~iOuY{?`1uD;082b?|oj|5>33YQ~l!V0&0+hrG6~Wz_S|Gm)HncJ` nEyJuGGz`_6kEfxVS literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/levels b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/levels new file mode 100644 index 0000000000000000000000000000000000000000..23ecc55259e97ef90d339c668eb4de3ffd005aff GIT binary patch literal 41 TcmeYX_GMycU|?VbV!QwVCU^k% literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/manifest b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/partitions/default2/manifest new file mode 100644 index 0000000000000000000000000000000000000000..445fd09222e5aa0f191025b0b5bba2d4bb9a6b3d GIT binary patch literal 7 OcmeYX_GMyZU2LN3~C?fy> literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/version b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/version new file mode 100644 index 000000000..29e6c1aa3 --- /dev/null +++ b/crates/fjall/test_fixture/v2_keyspace_corrupt_journal/version @@ -0,0 +1 @@ +FJL \ No newline at end of file diff --git a/crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/0.sealed b/crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/0.sealed new file mode 100644 index 0000000000000000000000000000000000000000..4a0a05eadc06aebb20dc3409b564d248b99689a0 GIT binary patch literal 46 tcmZQ%U|?VbVklr@U{6U+ODxSPVPH%IiZLcKmmTq%5|QCC$<51$2>^L02uJ_` literal 0 HcmV?d00001 diff --git a/crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/1.sealed b/crates/fjall/test_fixture/v2_sealed_journal_shenanigans/journals/1.sealed new file mode 100644 index 0000000000000000000000000000000000000000..88a33b123a0057b1b62e24b31e71b82f478cfe7b GIT binary patch literal 46 ucmZQ%U|?VbVh{k*ObqNPscDI&IVB8?Ng$yl=A3 fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + let mut batch = db.batch(); + + assert_eq!(tree.len()?, 0); + batch.insert(&tree, "1", "abc"); + batch.insert(&tree, "3", "abc"); + batch.insert(&tree, "5", "abc"); + assert_eq!(tree.len()?, 0); + + batch.commit()?; + assert_eq!(tree.len()?, 3); + + Ok(()) +} + +#[test] +fn blob_batch_simple() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Some(KvSeparationOptions::default())) + })?; + + let blob = "oxygen".repeat(128_000); + + let mut batch = db.batch(); + + assert_eq!(tree.len()?, 0); + batch.insert(&tree, "1", &blob); + batch.insert(&tree, "3", "abc"); + batch.insert(&tree, "5", "abc"); + assert_eq!(tree.len()?, 0); + + batch.commit()?; + assert_eq!(tree.len()?, 3); + + assert_eq!(&*tree.get("1")?.unwrap(), blob.as_bytes()); + + Ok(()) +} + +#[test] +fn batch_multi_keys() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + let mut batch = db.batch(); + + assert_eq!(tree.len()?, 0); + batch.insert(&tree, "1", "abc"); + batch.insert(&tree, "1", "def"); + batch.insert(&tree, "1", "ghi"); + assert_eq!(tree.len()?, 0); + + batch.commit()?; + assert_eq!(tree.len()?, 1); + assert_eq!(&*tree.get("1")?.unwrap(), b"ghi"); + + Ok(()) +} diff --git a/crates/fjall/tests/batch_recovery.rs b/crates/fjall/tests/batch_recovery.rs new file mode 100644 index 000000000..8f12ff9b4 --- /dev/null +++ b/crates/fjall/tests/batch_recovery.rs @@ -0,0 +1,21 @@ +use fjall::Database; +use test_log::test; + +#[test] +fn batch_recovery() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + for i in 0_u128..25 { + let db = Database::create_or_recover(Database::builder(folder.path()).into_config())?; + + let tree = db.keyspace("default", Default::default)?; + let tree2 = db.keyspace("default2", Default::default)?; + + let mut batch = db.batch(); + batch.insert(&tree, i.to_be_bytes(), i.to_be_bytes()); + batch.insert(&tree2, i.to_be_bytes(), i.to_be_bytes()); + batch.commit()?; + } + + Ok(()) +} diff --git a/crates/fjall/tests/blob_kv_simple.rs b/crates/fjall/tests/blob_kv_simple.rs new file mode 100644 index 000000000..f83a7ea84 --- /dev/null +++ b/crates/fjall/tests/blob_kv_simple.rs @@ -0,0 +1,63 @@ +use fjall::{Database, KeyspaceCreateOptions, KvSeparationOptions}; +use lsm_tree::AbstractTree; +use test_log::test; + +#[test] +fn blob_kv_simple() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Some(KvSeparationOptions::default())) + })?; + + assert_eq!(tree.len()?, 0); + tree.insert("1", "oxygen".repeat(1_000_000))?; + tree.insert("3", "abc")?; + tree.insert("5", "abc")?; + assert_eq!(tree.len()?, 3); + + tree.rotate_memtable_and_wait()?; + + if let fjall::AnyTree::Blob(tree) = &tree.tree { + assert!(tree.index.disk_space() < 1_000); + + // NOTE: The data is compressed quite well, so it's way less than 1M + assert!(tree.disk_space() > 5_000); + + assert_eq!(1, tree.blob_file_count()); + } else { + unreachable!(); + } + + Ok(()) +} + +#[test] +fn blob_kv_simple_recovery() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default() + .with_kv_separation(Some(KvSeparationOptions::default())) + })?; + + assert_eq!(tree.len()?, 0); + tree.insert("1", "oxygen".repeat(1_000_000))?; + tree.insert("3", "abc")?; + tree.insert("5", "abc")?; + assert_eq!(tree.len()?, 3); + + tree.rotate_memtable_and_wait()?; + } + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert_eq!(tree.len()?, 3); + } + + Ok(()) +} diff --git a/crates/fjall/tests/clear_dirty_read.rs b/crates/fjall/tests/clear_dirty_read.rs new file mode 100644 index 000000000..4f58ae390 --- /dev/null +++ b/crates/fjall/tests/clear_dirty_read.rs @@ -0,0 +1,35 @@ +use fjall::Readable; + +#[test_log::test] +fn clear_dirty_read() -> fjall::Result<()> { + use fjall::{KeyspaceCreateOptions, SingleWriterTxDatabase}; + + let folder = tempfile::tempdir()?; + + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert!(!tree.contains_key("a")?); + + let mut tx = db.write_tx(); + tx.insert(&tree, "a", "a"); + tx.commit()?; + + let snapshot = db.read_tx(); + assert!(snapshot.contains_key(&tree, "a")?); + + tree.inner().clear()?; + + assert!(snapshot.contains_key(&tree, "a")?); + + assert!(!tree.contains_key("a")?); + assert_eq!(0, tree.approximate_len()); + + { + let snapshot = db.read_tx(); + assert!(!snapshot.contains_key(&tree, "a")?); + } + + Ok(()) +} diff --git a/crates/fjall/tests/compaction_filter.rs b/crates/fjall/tests/compaction_filter.rs new file mode 100644 index 000000000..7b01cef8d --- /dev/null +++ b/crates/fjall/tests/compaction_filter.rs @@ -0,0 +1,158 @@ +use fjall::KeyspaceCreateOptions; +use lsm_tree::compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, +}; +use lsm_tree::get_tmp_folder; +use std::sync::Arc; +use test_log::test; + +/// Only keeps KVs that start with "a" +struct AFilter; + +impl CompactionFilter for AFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key().starts_with(b"a") { + Ok(Verdict::Keep) + } else { + Ok(Verdict::Remove) + } + } +} + +struct MyFactory; + +impl Factory for MyFactory { + fn name(&self) -> &str { + "A" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(AFilter) + } +} + +#[test] +fn compaction_filter() -> fjall::Result<()> { + let folder = get_tmp_folder(); + + let db = fjall::Database::builder(&folder) + .with_compaction_filter_factories(Arc::new(|keyspace| match keyspace { + "my_items" => Some(Arc::new(MyFactory)), + _ => None, + })) + .open()?; + + { + let tree = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + + tree.insert("a", "a")?; + tree.rotate_memtable_and_wait()?; + tree.insert("abc", "abc")?; + tree.insert("b", "b")?; + tree.rotate_memtable_and_wait()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(tree.contains_key("b")?); + + tree.major_compact()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(!tree.contains_key("b")?); + } + + { + let tree = db.keyspace("my_items_imposter", KeyspaceCreateOptions::default)?; + + tree.insert("a", "a")?; + tree.rotate_memtable_and_wait()?; + tree.insert("abc", "abc")?; + tree.insert("b", "b")?; + tree.rotate_memtable_and_wait()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(tree.contains_key("b")?); + + tree.major_compact()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(tree.contains_key("b")?); + } + + Ok(()) +} + +#[test] +fn compaction_filter_recover() -> fjall::Result<()> { + let folder = get_tmp_folder(); + + { + let db = fjall::Database::builder(&folder) + .with_compaction_filter_factories(Arc::new(|keyspace| match keyspace { + "my_items" => Some(Arc::new(MyFactory)), + _ => None, + })) + .open()?; + + let _tree = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + let _tree = db.keyspace("my_items_imposter", KeyspaceCreateOptions::default)?; + } + + { + let db = fjall::Database::builder(&folder) + .with_compaction_filter_factories(Arc::new(|keyspace| match keyspace { + "my_items" => Some(Arc::new(MyFactory)), + _ => None, + })) + .open()?; + + { + let tree = db.keyspace("my_items", KeyspaceCreateOptions::default)?; + + tree.insert("a", "a")?; + tree.rotate_memtable_and_wait()?; + tree.insert("abc", "abc")?; + tree.insert("b", "b")?; + tree.rotate_memtable_and_wait()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(tree.contains_key("b")?); + + tree.major_compact()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(!tree.contains_key("b")?); + } + + { + let tree = db.keyspace("my_items_imposter", KeyspaceCreateOptions::default)?; + + tree.insert("a", "a")?; + tree.rotate_memtable_and_wait()?; + tree.insert("abc", "abc")?; + tree.insert("b", "b")?; + tree.rotate_memtable_and_wait()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(tree.contains_key("b")?); + + tree.major_compact()?; + + assert!(tree.contains_key("a")?); + assert!(tree.contains_key("abc")?); + assert!(tree.contains_key("b")?); + } + } + + Ok(()) +} diff --git a/crates/fjall/tests/db_lock.rs b/crates/fjall/tests/db_lock.rs new file mode 100644 index 000000000..0c4cc2872 --- /dev/null +++ b/crates/fjall/tests/db_lock.rs @@ -0,0 +1,23 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +#[test] +fn db_lock() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + tree.insert("asd", "def")?; + tree.insert("efg", "hgf")?; + tree.insert("hij", "wer")?; + + drop(db); + + assert!(matches!( + Database::builder(&folder).open(), + Err(fjall::Error::Locked), + )); + + Ok(()) +} diff --git a/crates/fjall/tests/db_open.rs b/crates/fjall/tests/db_open.rs new file mode 100644 index 000000000..f5df96620 --- /dev/null +++ b/crates/fjall/tests/db_open.rs @@ -0,0 +1,34 @@ +use fjall::Database; + +#[test_log::test] +fn db_open() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let _db = Database::builder(&folder).open()?; + } + + // DB should not be locked + { + let _db = Database::builder(&folder).open()?; + } + + Ok(()) +} + +#[test_log::test] +fn db_open_with_keyspace() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + let _keyspace = db.keyspace("hello", Default::default)?; + } + + // DB should not be locked + { + let _db = Database::builder(&folder).open()?; + } + + Ok(()) +} diff --git a/crates/fjall/tests/fifo_dirty_read.rs b/crates/fjall/tests/fifo_dirty_read.rs new file mode 100644 index 000000000..747bf9068 --- /dev/null +++ b/crates/fjall/tests/fifo_dirty_read.rs @@ -0,0 +1,31 @@ +use fjall::Readable; + +#[test_log::test] +fn fifo_dirty_read() -> fjall::Result<()> { + use fjall::{KeyspaceCreateOptions, SingleWriterTxDatabase}; + use std::sync::Arc; + + let folder = tempfile::tempdir()?; + + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default() + .compaction_strategy(Arc::new(fjall::compaction::Fifo::new(1, None))) + })?; + + assert!(!tree.contains_key("a")?); + + let mut tx = db.write_tx(); + tx.insert(&tree, "a", "a"); + tx.commit()?; + + let snapshot = db.read_tx(); + assert!(snapshot.contains_key(&tree, "a")?); + + tree.inner().rotate_memtable_and_wait()?; + + assert!(snapshot.contains_key(&tree, "a")?); + + Ok(()) +} diff --git a/crates/fjall/tests/format_v4.rs b/crates/fjall/tests/format_v4.rs new file mode 100644 index 000000000..1539bdb75 --- /dev/null +++ b/crates/fjall/tests/format_v4.rs @@ -0,0 +1,41 @@ +use fjall::{Database, Error, FormatVersion, KeyspaceCreateOptions}; + +#[test] +fn new_database_uses_v4_and_reopens_fixed_width_tables() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let database = Database::builder(folder.path()).open()?; + let keyspace = database.keyspace("fixed", KeyspaceCreateOptions::default)?; + let mut ingestion = keyspace.start_ingestion()?; + for index in 0..1_000_u64 { + ingestion.write(index.to_be_bytes(), (index as u32).to_be_bytes())?; + } + ingestion.finish_exclusive()?; + } + + assert_eq!( + std::fs::read(folder.path().join("version"))?, + [b'F', b'J', b'L', 4], + ); + + let database = Database::builder(folder.path()).open()?; + let keyspace = database.keyspace("fixed", KeyspaceCreateOptions::default)?; + assert_eq!( + keyspace.get(777_u64.to_be_bytes())?.as_deref(), + Some(777_u32.to_be_bytes().as_slice()), + ); + Ok(()) +} + +#[test] +fn v3_database_requires_resync() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + std::fs::write(folder.path().join("version"), [b'F', b'J', b'L', 3])?; + + assert!(matches!( + Database::builder(folder.path()).open(), + Err(Error::InvalidVersion(Some(FormatVersion::V3))), + )); + Ok(()) +} diff --git a/crates/fjall/tests/ingest_recovery.rs b/crates/fjall/tests/ingest_recovery.rs new file mode 100644 index 000000000..d75f0cdf0 --- /dev/null +++ b/crates/fjall/tests/ingest_recovery.rs @@ -0,0 +1,73 @@ +use fjall::{Database, Readable, Slice}; +use test_log::test; + +#[test] +fn ingest_recovery() -> fjall::Result<()> { + let path = tempfile::tempdir()?; + + let ks = "default"; + let key: Slice = b"abc".into(); + let value: Slice = b"zzz".into(); + + { + let db = Database::builder(&path).open()?; + let keyspace = db.keyspace(ks, Default::default)?; + let mut ing = keyspace.start_ingestion()?; + ing.write(key.clone(), value.clone())?; + ing.finish()?; + assert_eq!(keyspace.get(key.clone())?, Some(value.clone())); // ok + } + + { + let db = Database::builder(&path).open()?; + let keyspace = db.keyspace(ks, Default::default)?; + assert_eq!(keyspace.get(key.clone())?, Some(value.clone())); // ok + } + + { + let db = Database::builder(&path).open()?; + let keyspace = db.keyspace(ks, Default::default)?; + let snapshot = db.snapshot(); + assert_eq!(snapshot.get(&keyspace, key.clone())?, Some(value.clone())); // snapshot - not ok + } + + Ok(()) +} + +#[test] +fn concurrent_exclusive_ingest_recovery() -> fjall::Result<()> { + let path = tempfile::tempdir()?; + + { + let db = Database::builder(&path).open()?; + let first = db.keyspace("first", Default::default)?; + let second = db.keyspace("second", Default::default)?; + + let (first_result, second_result) = std::thread::scope(|scope| { + let first = scope.spawn(|| { + let mut ingestion = first.start_ingestion()?; + ingestion.write(b"a", b"first")?; + ingestion.finish_exclusive() + }); + let second = scope.spawn(|| { + let mut ingestion = second.start_ingestion()?; + ingestion.write(b"b", b"second")?; + ingestion.finish_exclusive() + }); + + (first.join().unwrap(), second.join().unwrap()) + }); + first_result?; + second_result?; + } + + { + let db = Database::builder(&path).open()?; + let first = db.keyspace("first", Default::default)?; + let second = db.keyspace("second", Default::default)?; + assert_eq!(first.get(b"a")?.as_deref(), Some(b"first".as_slice())); + assert_eq!(second.get(b"b")?.as_deref(), Some(b"second".as_slice())); + } + + Ok(()) +} diff --git a/crates/fjall/tests/journal_large_value.rs b/crates/fjall/tests/journal_large_value.rs new file mode 100644 index 000000000..d94fc711d --- /dev/null +++ b/crates/fjall/tests/journal_large_value.rs @@ -0,0 +1,55 @@ +// Regression test for https://github.com/fjall-rs/fjall/issues/68 + +use fjall::{Database, KeyspaceCreateOptions, KvSeparationOptions}; + +#[test_log::test] +fn journal_recover_large_value() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let large_value = "a".repeat(128_000); + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + tree.insert("a", &large_value)?; + tree.insert("b", "b")?; + } + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert_eq!(large_value.as_bytes(), &*tree.get("a")?.unwrap()); + assert_eq!(b"b", &*tree.get("b")?.unwrap()); + } + + Ok(()) +} + +#[test_log::test] +fn journal_recover_large_value_blob() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let large_value = "a".repeat(128_000); + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default() + .with_kv_separation(Some(KvSeparationOptions::default())) + })?; + tree.insert("a", &large_value)?; + tree.insert("b", "b")?; + } + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default() + .with_kv_separation(Some(KvSeparationOptions::default())) + })?; + assert_eq!(large_value.as_bytes(), &*tree.get("a")?.unwrap()); + assert_eq!(b"b", &*tree.get("b")?.unwrap()); + } + + Ok(()) +} diff --git a/crates/fjall/tests/keyspace_clear.rs b/crates/fjall/tests/keyspace_clear.rs new file mode 100644 index 000000000..2eb68752e --- /dev/null +++ b/crates/fjall/tests/keyspace_clear.rs @@ -0,0 +1,84 @@ +use fjall::{Database, KeyspaceCreateOptions}; + +#[test_log::test] +fn clear_recover() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert!(tree.is_empty()?); + + tree.insert("a", "a")?; + assert!(tree.contains_key("a")?); + + tree.clear()?; + assert!(tree.is_empty()?); + } + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert!(tree.is_empty()?); + } + + Ok(()) +} + +#[test_log::test] +fn clear_recover_multi_tree() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + let other = db.keyspace("other", KeyspaceCreateOptions::default)?; + assert!(tree.is_empty()?); + + tree.insert("a", "a")?; + assert!(tree.contains_key("a")?); + + tree.clear()?; + assert!(tree.is_empty()?); + + other.insert("a", "z")?; + } + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert!(tree.is_empty()?); + + let other = db.keyspace("other", KeyspaceCreateOptions::default)?; + other.clear()?; + assert!(other.is_empty()?); + } + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + let other = db.keyspace("other", KeyspaceCreateOptions::default)?; + assert!(tree.is_empty()?); + assert!(other.is_empty()?); + + tree.insert("a", "a")?; + other.clear()?; + } + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + let other = db.keyspace("other", KeyspaceCreateOptions::default)?; + assert!(tree.contains_key("a")?); + assert!(other.is_empty()?); + } + + Ok(()) +} diff --git a/crates/fjall/tests/keyspace_iter_lifetime.rs b/crates/fjall/tests/keyspace_iter_lifetime.rs new file mode 100644 index 000000000..6075939c4 --- /dev/null +++ b/crates/fjall/tests/keyspace_iter_lifetime.rs @@ -0,0 +1,34 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +struct Counter<'a, I: DoubleEndedIterator>> { + iter: &'a mut I, +} + +impl<'a, I: DoubleEndedIterator>> Counter<'a, I> { + pub fn execute(self) -> usize { + self.iter.count() + } +} + +#[test] +fn keyspace_iter_lifetime() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert_eq!(0, db.write_buffer_size()); + + tree.insert("asd", "def")?; + tree.insert("efg", "hgf")?; + tree.insert("hij", "wer")?; + + { + let mut iter = tree.iter().map(|guard| guard.into_inner()); + let counter = Counter { iter: &mut iter }; + assert_eq!(3, counter.execute()); + } + + Ok(()) +} diff --git a/crates/fjall/tests/keyspace_recover.rs b/crates/fjall/tests/keyspace_recover.rs new file mode 100644 index 000000000..960688102 --- /dev/null +++ b/crates/fjall/tests/keyspace_recover.rs @@ -0,0 +1,290 @@ +use fjall::config::{ + BlockSizePolicy, FilterPolicy, FilterPolicyEntry, HashRatioPolicy, PinningPolicy, + RestartIntervalPolicy, +}; +use fjall::{AbstractTree, CompressionType, Database, KeyspaceCreateOptions, KvSeparationOptions}; +use lsm_tree::compaction::CompactionStrategy; +use std::sync::Arc; +use test_log::test; + +const ITEM_COUNT: usize = 100; + +#[test] +fn reload_keyspace_config_fifo() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let strategy = fjall::compaction::Fifo::new(555, Some(6)); + + let expected_kvs = strategy.get_config(); + + { + let db = Database::builder(&folder).open()?; + + let _tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().compaction_strategy(Arc::new(strategy)) + })?; + }; + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert_eq!( + expected_kvs, + tree.config.compaction_strategy.get_config(), + "compaction strategy config does not match", + ); + } + + Ok(()) +} + +#[test] +fn reload_keyspace_config_leveled() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let strategy = fjall::compaction::Leveled::default() + .with_l0_threshold(6) + .with_level_ratio_policy(vec![4.0, 6.0, 8.0]) + .with_table_target_size(100_000_000); + + let expected_kvs = strategy.get_config(); + + { + let db = Database::builder(&folder).open()?; + + let _tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().compaction_strategy(Arc::new(strategy)) + })?; + }; + + { + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert_eq!( + expected_kvs, + tree.config.compaction_strategy.get_config(), + "compaction strategy config does not match", + ); + } + + Ok(()) +} + +#[test] +fn reload_keyspace_config() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let data_block_size = BlockSizePolicy::all(6_666); + + let data_block_interval_policy = RestartIntervalPolicy::all(8); + // let index_block_interval_policy = RestartIntervalPolicy::all(9); + + let filter_block_pinning_policy = PinningPolicy::new([true, true, true, false]); + let index_block_pinning_policy = PinningPolicy::new([true, true, false]); + + let filter_block_partitioning_policy = PinningPolicy::new([false, false, true]); + let index_block_partitioning_policy = PinningPolicy::new([false, false, false, false, true]); + + let filter_policy = FilterPolicy::new([FilterPolicyEntry::Bloom( + fjall::config::BloomConstructionPolicy::FalsePositiveRate(0.54321), + )]); + + let data_block_hash_ratio_policy = HashRatioPolicy::all(0.5); + + { + let db = Database::builder(&folder).open()?; + + let keyspace = db.keyspace("default", || { + KeyspaceCreateOptions::default() + .data_block_size_policy(data_block_size.clone()) + .data_block_restart_interval_policy(data_block_interval_policy.clone()) + // .index_block_restart_interval_policy(index_block_policy.clone()) + .filter_block_pinning_policy(filter_block_pinning_policy.clone()) + .index_block_pinning_policy(index_block_pinning_policy.clone()) + .filter_block_partitioning_policy(filter_block_partitioning_policy.clone()) + .index_block_partitioning_policy(index_block_partitioning_policy.clone()) + .expect_point_read_hits(true) + .filter_policy(filter_policy.clone()) + .data_block_hash_ratio_policy(data_block_hash_ratio_policy.clone()) + })?; + + let real_config = keyspace.tree.tree_config(); + + assert_eq!(data_block_size, real_config.data_block_size_policy); + assert_eq!( + data_block_interval_policy, + real_config.data_block_restart_interval_policy, + ); + // assert_eq!( + // index_block_interval_policy, + // tree.config.index_block_restart_interval_policy, + // ); + assert_eq!( + filter_block_partitioning_policy, + real_config.filter_block_partitioning_policy, + ); + assert_eq!( + filter_block_pinning_policy, + real_config.filter_block_pinning_policy, + ); + assert_eq!( + index_block_partitioning_policy, + real_config.index_block_partitioning_policy, + ); + assert_eq!( + index_block_pinning_policy, + real_config.index_block_pinning_policy, + ); + + assert_eq!(filter_policy, real_config.filter_policy); + + assert_eq!( + data_block_hash_ratio_policy, + real_config.data_block_hash_ratio_policy, + ); + + assert!(keyspace.config.expect_point_read_hits); + }; + + { + let db = Database::builder(&folder).open()?; + let keyspace = db.keyspace("default", || unreachable!())?; + + let real_config = keyspace.tree.tree_config(); + + assert_eq!(data_block_size, real_config.data_block_size_policy); + assert_eq!( + data_block_interval_policy, + real_config.data_block_restart_interval_policy, + ); + // assert_eq!( + // index_block_interval_policy, + // tree.config.index_block_restart_interval_policy, + // ); + assert_eq!( + filter_block_partitioning_policy, + real_config.filter_block_partitioning_policy, + ); + assert_eq!( + filter_block_pinning_policy, + real_config.filter_block_pinning_policy, + ); + assert_eq!( + index_block_partitioning_policy, + real_config.index_block_partitioning_policy, + ); + assert_eq!( + index_block_pinning_policy, + real_config.index_block_pinning_policy, + ); + + assert_eq!(filter_policy, real_config.filter_policy); + + assert_eq!( + data_block_hash_ratio_policy, + real_config.data_block_hash_ratio_policy, + ); + + assert!(keyspace.config.expect_point_read_hits); + } + + Ok(()) +} + +#[test] +fn reload_keyspace_config_blob_opts() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let _tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Some( + KvSeparationOptions::default() + .age_cutoff(0.55) + .compression(CompressionType::None) + .file_target_size(124) + .separation_threshold(515) + .staleness_threshold(0.77), + )) + })?; + }; + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + let blob_opts = tree.config.kv_separation_opts.as_ref().unwrap(); + + assert_eq!(blob_opts.compression, CompressionType::None); + assert_eq!(blob_opts.age_cutoff, 0.55); + assert_eq!(blob_opts.file_target_size, 124); + assert_eq!(blob_opts.separation_threshold, 515); + assert_eq!(blob_opts.staleness_threshold, 0.77); + } + + Ok(()) +} + +#[test] +fn reload_with_keyspaces() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + + let keyspaces = &[ + db.keyspace("default1", KeyspaceCreateOptions::default)?, + db.keyspace("default2", KeyspaceCreateOptions::default)?, + db.keyspace("default3", KeyspaceCreateOptions::default)?, + ]; + + for tree in keyspaces { + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + } + + for tree in keyspaces { + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(|x| x.key()).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + } + } + + for _ in 0..10 { + let db = Database::builder(&folder).open()?; + + let keyspaces = &[ + db.keyspace("default1", KeyspaceCreateOptions::default)?, + db.keyspace("default2", KeyspaceCreateOptions::default)?, + db.keyspace("default3", KeyspaceCreateOptions::default)?, + ]; + + for tree in keyspaces { + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(|x| x.key()).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + } + } + + Ok(()) +} diff --git a/crates/fjall/tests/keyspace_snapshot.rs b/crates/fjall/tests/keyspace_snapshot.rs new file mode 100644 index 000000000..fe2371fff --- /dev/null +++ b/crates/fjall/tests/keyspace_snapshot.rs @@ -0,0 +1,108 @@ +use fjall::{Database, KeyspaceCreateOptions, Readable}; +use test_log::test; + +#[test] +fn keyspace_iter_dirty_read() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Default::default()) + })?; + + tree.insert("a#1", "a")?; + tree.insert("a#2", "b")?; + tree.insert("a#3", "c")?; + + tree.rotate_memtable_and_wait()?; + + let iter = tree.iter(); + + tree.insert("a#4", "d")?; + + assert_eq!(3, iter.count()); + + Ok(()) +} + +#[test] +fn keyspace_snapshot_read() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Default::default()) + })?; + let tree2 = db.keyspace("default2", || { + KeyspaceCreateOptions::default().with_kv_separation(Default::default()) + })?; + + tree.insert("a#1", "a")?; + tree.insert("a#2", "b")?; + tree.insert("a#3", "c")?; + tree2.insert("b#1", "5")?; + + let snapshot = db.snapshot(); + + tree.rotate_memtable_and_wait()?; + + tree.insert("a#4", "d")?; + tree2.insert("b#2", "6")?; + + assert_eq!(3, snapshot.iter(&tree).count()); + assert_eq!(3, snapshot.iter(&tree).count()); + + Ok(()) +} + +#[test] +fn keyspace_visible_seqno() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + let initial_seqno = db.snapshot().seqno(); + + tree.insert("a#1", "a")?; + + let snapshot = db.snapshot(); + assert_eq!(initial_seqno + 1, snapshot.seqno()); + + tree.insert("a#2", "a")?; + + let snapshot = db.snapshot(); + assert_eq!(initial_seqno + 2, snapshot.seqno()); + + Ok(()) +} + +#[test] +fn keyspace_torn_read() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let initial_seqno = db.snapshot().seqno(); + + // Fake start of a batch + let batch_seqno = db.supervisor.seqno.next(); + + // ... + + let snapshot = db.snapshot(); + assert_eq!(initial_seqno, snapshot.seqno()); + + // ... + + // Submitting batch completion + db.supervisor.snapshot_tracker.publish(batch_seqno); + + let snapshot = db.snapshot(); + assert_eq!(initial_seqno + 1, snapshot.seqno()); + + Ok(()) +} diff --git a/crates/fjall/tests/keyspace_v1_load_fixture.rs b/crates/fjall/tests/keyspace_v1_load_fixture.rs new file mode 100644 index 000000000..273ab6f55 --- /dev/null +++ b/crates/fjall/tests/keyspace_v1_load_fixture.rs @@ -0,0 +1,30 @@ +use fjall::Database; +use test_log::test; + +#[test] +fn db_load_v1() -> fjall::Result<()> { + let folder = "test_fixture/v1_keyspace"; + + let result = Database::builder(folder).open(); + + matches!( + result, + Err(fjall::Error::InvalidVersion(Some(fjall::FormatVersion::V1))) + ); + + Ok(()) +} + +#[test] +fn db_load_v1_corrupt_journal() -> fjall::Result<()> { + let folder = "test_fixture/v1_keyspace_corrupt_journal"; + + let result = Database::builder(folder).open(); + + matches!( + result, + Err(fjall::Error::InvalidVersion(Some(fjall::FormatVersion::V1))) + ); + + Ok(()) +} diff --git a/crates/fjall/tests/keyspace_v2_load_fixture.rs b/crates/fjall/tests/keyspace_v2_load_fixture.rs new file mode 100644 index 000000000..aa8480bd1 --- /dev/null +++ b/crates/fjall/tests/keyspace_v2_load_fixture.rs @@ -0,0 +1,32 @@ +use fjall::{Database, FormatVersion, JournalRecoveryError}; +use test_log::test; + +#[test] +fn db_load_v2() -> fjall::Result<()> { + let folder = "test_fixture/v2_keyspace"; + + let result = Database::builder(folder).open(); + + matches!( + result, + Err(fjall::Error::InvalidVersion(Some(FormatVersion::V2))) + ); + + Ok(()) +} + +#[test] +fn db_load_v2_corrupt_journal() -> fjall::Result<()> { + let folder = "test_fixture/v2_keyspace_corrupt_journal"; + + let result = Database::builder(folder).open(); + + matches!( + result, + Err(fjall::Error::JournalRecovery( + JournalRecoveryError::ChecksumMismatch + )), + ); + + Ok(()) +} diff --git a/crates/fjall/tests/memtable_recover.rs b/crates/fjall/tests/memtable_recover.rs new file mode 100644 index 000000000..b2c67ba94 --- /dev/null +++ b/crates/fjall/tests/memtable_recover.rs @@ -0,0 +1,47 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +const ITEM_COUNT: usize = 10_000; + +#[test] +fn reload_with_memtable() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + } + + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(|x| x.key()).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + } + + for _ in 0..5 { + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(|x| x.key()).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + } + + Ok(()) +} diff --git a/crates/fjall/tests/prefix_complex.rs b/crates/fjall/tests/prefix_complex.rs new file mode 100644 index 000000000..f46e4c2a6 --- /dev/null +++ b/crates/fjall/tests/prefix_complex.rs @@ -0,0 +1,45 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +#[test] +fn keyspace_prefix_carl() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Default::default()) + })?; + + tree.insert("a#1", "a".repeat(4_096))?; + tree.insert("a#2", "b".repeat(4_096))?; + tree.insert("a#3", "c".repeat(4_096))?; + + tree.rotate_memtable_and_wait()?; + + { + let mut iter = tree.prefix("a#"); + + let guard = iter.next().expect("should exist"); + let key = guard.key()?; // does not load blob + assert_eq!(b"a#1", &*key); + + let guard = iter.next().expect("should exist"); + let (key, value) = guard.into_inner()?; // loads blob + assert_eq!( + (b"a#2" as &[u8], &b"b".repeat(4_096) as &[u8]), + (&*key, &*value), + ); + + let guard = iter.next().expect("should exist"); + let (key, value) = guard.into_inner()?; // loads blob + assert_eq!( + (b"a#3" as &[u8], &b"c".repeat(4_096) as &[u8]), + (&*key, &*value), + ); + + assert!(iter.next().is_none(), "iter should be consumed"); + } + + Ok(()) +} diff --git a/crates/fjall/tests/recover_from_different_folder.rs b/crates/fjall/tests/recover_from_different_folder.rs new file mode 100644 index 000000000..f4c5f3483 --- /dev/null +++ b/crates/fjall/tests/recover_from_different_folder.rs @@ -0,0 +1,35 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +#[test] +fn recover_from_different_folder() -> fjall::Result<()> { + if std::path::Path::new(".test").try_exists()? { + std::fs::remove_dir_all(".test")?; + } + + let folder = ".test/asd"; + + { + let db = Database::builder(folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + tree.insert("abc", "def")?; + tree.insert("wqewe", "def")?; + tree.insert("ewewq", "def")?; + tree.insert("asddas", "def")?; + tree.insert("ycxycx", "def")?; + tree.insert("asdsda", "def")?; + tree.insert("wewqe", "def")?; + } + + let absolute_folder = std::path::Path::new(folder).canonicalize()?; + + std::fs::create_dir_all(".test/def")?; + std::env::set_current_dir(".test/def")?; + + for _ in 0..100 { + let _db = Database::builder(&absolute_folder).open()?; + } + + Ok(()) +} diff --git a/crates/fjall/tests/recovery_journal_mac.rs b/crates/fjall/tests/recovery_journal_mac.rs new file mode 100644 index 000000000..09eeff954 --- /dev/null +++ b/crates/fjall/tests/recovery_journal_mac.rs @@ -0,0 +1,36 @@ +use fjall::Database; +use test_log::test; + +#[test] +fn db_journal_recovery_mac_ds_store() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let _db = Database::builder(&folder).open()?; + } + + std::fs::File::create(folder.path().join(".DS_Store"))?; + + { + let _db = Database::builder(&folder).open()?; + } + + Ok(()) +} + +#[test] +fn db_journal_recovery_mac_underscore_file() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let _db = Database::builder(&folder).open()?; + } + + std::fs::File::create(folder.path().join("._0"))?; + + { + let _db = Database::builder(&folder).open()?; + } + + Ok(()) +} diff --git a/crates/fjall/tests/recovery_keyspaces_mac.rs b/crates/fjall/tests/recovery_keyspaces_mac.rs new file mode 100644 index 000000000..00f739396 --- /dev/null +++ b/crates/fjall/tests/recovery_keyspaces_mac.rs @@ -0,0 +1,36 @@ +use fjall::Database; +use test_log::test; + +#[test] +fn db_keyspaces_recovery_mac_ds_store() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let _db = Database::builder(&folder).open()?; + } + + std::fs::File::create(folder.path().join("keyspaces").join(".DS_Store"))?; + + { + let _db = Database::builder(&folder).open()?; + } + + Ok(()) +} + +#[test] +fn db_keyspaces_recovery_mac_underscore() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + { + let _db = Database::builder(&folder).open()?; + } + + std::fs::File::create(folder.path().join("keyspaces").join("._0"))?; + + { + let _db = Database::builder(&folder).open()?; + } + + Ok(()) +} diff --git a/crates/fjall/tests/seqno_recovery.rs b/crates/fjall/tests/seqno_recovery.rs new file mode 100644 index 000000000..08acfaafb --- /dev/null +++ b/crates/fjall/tests/seqno_recovery.rs @@ -0,0 +1,155 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +const ITEM_COUNT: usize = 100; + +#[test] +fn recover_seqno_smoke_test() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let seqno; + + { + let db = Database::builder(&folder).open()?; + + let keyspace = db.keyspace("default1", KeyspaceCreateOptions::default)?; + + for k in ["a", "b", "c"] { + keyspace.insert(k, k)?; + } + + seqno = db.seqno(); + } + + for _ in 0..10 { + let db = Database::builder(&folder).open()?; + assert_eq!(seqno, db.seqno()); + assert_eq!(seqno, db.visible_seqno()); + + let keyspace = db.keyspace("default1", KeyspaceCreateOptions::default)?; + + for k in ["a", "b", "c"] { + keyspace.get(k)?.unwrap(); + } + } + + Ok(()) +} + +#[test] +fn recover_seqno() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let mut seqno; + + { + let db = Database::builder(&folder).open()?; + + let keyspaces = &[ + db.keyspace("default1", KeyspaceCreateOptions::default)?, + db.keyspace("default2", KeyspaceCreateOptions::default)?, + db.keyspace("default3", KeyspaceCreateOptions::default)?, + ]; + + // NOTE: Opening a keyspace increments the global seqno + seqno = db.seqno(); + + for tree in keyspaces { + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + + seqno += 1; + assert_eq!(seqno, db.seqno()); + } + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes())?; + + seqno += 1; + assert_eq!(seqno, db.seqno()); + } + } + + for tree in keyspaces { + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(|x| x.key()).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(|x| x.key()).count(), + ITEM_COUNT * 2, + ); + } + } + + for _ in 0..10 { + let db = Database::builder(&folder).open()?; + assert_eq!(seqno, db.seqno()); + assert_eq!(seqno, db.visible_seqno()); + + let keyspaces = &[ + db.keyspace("default1", KeyspaceCreateOptions::default)?, + db.keyspace("default2", KeyspaceCreateOptions::default)?, + db.keyspace("default3", KeyspaceCreateOptions::default)?, + ]; + + for tree in keyspaces { + assert_eq!(tree.len()?, ITEM_COUNT * 2); + assert_eq!(tree.iter().flat_map(|x| x.key()).count(), ITEM_COUNT * 2); + assert_eq!( + tree.iter().rev().flat_map(|x| x.key()).count(), + ITEM_COUNT * 2, + ); + } + } + + Ok(()) +} + +#[test] +fn recover_seqno_tombstone() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let mut seqno; + + { + let db = Database::builder(&folder).open()?; + + let keyspaces = &[ + db.keyspace("default1", KeyspaceCreateOptions::default)?, + db.keyspace("default2", KeyspaceCreateOptions::default)?, + db.keyspace("default3", KeyspaceCreateOptions::default)?, + ]; + + // NOTE: Opening a keyspace increments the global seqno + seqno = db.seqno(); + + for tree in keyspaces { + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.remove(key)?; + + seqno += 1; + assert_eq!(seqno, db.seqno()); + } + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + tree.remove(key)?; + + seqno += 1; + assert_eq!(seqno, db.seqno()); + } + } + } + + for _ in 0..10 { + let db = Database::builder(&folder).open()?; + assert_eq!(seqno, db.seqno()); + assert_eq!(seqno, db.visible_seqno()); + } + + Ok(()) +} diff --git a/crates/fjall/tests/standard_keyspace.rs b/crates/fjall/tests/standard_keyspace.rs new file mode 100644 index 000000000..ac056c3a0 --- /dev/null +++ b/crates/fjall/tests/standard_keyspace.rs @@ -0,0 +1,62 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +fn pair(item: fjall::Result) -> fjall::Result<(Vec, Vec)> { + let (key, value) = item?; + Ok((key.to_vec(), value.to_vec())) +} + +#[test] +fn standard_fast_paths_preserve_results_and_snapshot() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + let db = Database::builder(&folder).open()?; + let tree = db.keyspace("standard", KeyspaceCreateOptions::default)?; + + let mut ingestion = tree.start_ingestion()?; + ingestion.write_prevalidated("a1", "one")?; + ingestion.write_prevalidated("a2", "two")?; + ingestion.write_prevalidated("b1", "three")?; + ingestion.write_prevalidated_weak_tombstone("c1")?; + ingestion.finish_exclusive()?; + + assert_eq!(tree.get_standard("a2")?.as_deref(), Some(b"two".as_slice())); + assert_eq!(tree.get_standard("c1")?, None); + + let all = tree + .iter_standard() + .map(pair) + .collect::>>()?; + assert_eq!( + all, + [ + (b"a1".to_vec(), b"one".to_vec()), + (b"a2".to_vec(), b"two".to_vec()), + (b"b1".to_vec(), b"three".to_vec()), + ] + ); + + let prefix = tree + .prefix_standard("a") + .map(pair) + .collect::>>()?; + assert_eq!(prefix.len(), 2); + + let reverse_range = tree + .range_standard("a2"..="b1") + .rev() + .map(pair) + .collect::>>()?; + assert_eq!( + reverse_range, + [ + (b"b1".to_vec(), b"three".to_vec()), + (b"a2".to_vec(), b"two".to_vec()), + ] + ); + + let snapshot_iter = tree.iter_standard(); + tree.insert("z1", "later")?; + assert_eq!(snapshot_iter.count(), 3); + + Ok(()) +} diff --git a/crates/fjall/tests/tx_ryow.rs b/crates/fjall/tests/tx_ryow.rs new file mode 100644 index 000000000..69e99d03e --- /dev/null +++ b/crates/fjall/tests/tx_ryow.rs @@ -0,0 +1,31 @@ +use fjall::Readable; + +#[test_log::test] +fn tx_ryow() -> fjall::Result<()> { + use fjall::{KeyspaceCreateOptions, SingleWriterTxDatabase}; + + let folder = tempfile::tempdir()?; + + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + let mut tx = db.write_tx(); + + assert!(!tx.contains_key(&tree, "a")?); + + tx.insert(&tree, "a", "a"); + assert!(tx.contains_key(&tree, "a")?); + + tx.remove(&tree, "a"); + assert!(!tx.contains_key(&tree, "a")?); + + tx.insert(&tree, "a", "a"); + tx.insert(&tree, "a", "c"); + assert_eq!(b"c", &*tx.get(&tree, "a")?.unwrap()); + + tx.remove(&tree, "a"); + assert!(!tx.contains_key(&tree, "a")?); + + Ok(()) +} diff --git a/crates/fjall/tests/tx_ryow_snapshot.rs b/crates/fjall/tests/tx_ryow_snapshot.rs new file mode 100644 index 000000000..7453fedad --- /dev/null +++ b/crates/fjall/tests/tx_ryow_snapshot.rs @@ -0,0 +1,59 @@ +use fjall::Readable; + +/// Tests that modifications to a write transaction do not dirty a previously started snapshot +#[test_log::test] +fn tx_ryow_snapshot() -> fjall::Result<()> { + use fjall::{KeyspaceCreateOptions, SingleWriterTxDatabase}; + + let folder = tempfile::tempdir()?; + + let db = SingleWriterTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + let mut tx = db.write_tx(); + + { + let iter = tx.iter(&tree); + for x in 0u64..100 { + tx.insert(&tree, x.to_be_bytes(), "a"); + } + assert_eq!(0, iter.count()); + } + + assert_eq!(100, tx.iter(&tree).count()); + + Ok(()) +} + +#[test_log::test] +fn tx_ryow_snapshot_ssi() -> fjall::Result<()> { + use fjall::{KeyspaceCreateOptions, OptimisticTxDatabase}; + + let folder = tempfile::tempdir()?; + + let db = OptimisticTxDatabase::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + let mut tx = db.write_tx()?; + + { + let iter = tx.iter(&tree); + for x in 0u64..100 { + tx.insert(&tree, x.to_be_bytes(), "a"); + } + assert_eq!(0, iter.count()); + } + + assert_eq!(100, tx.iter(&tree).count()); + + tree.insert("2", "2")?; + assert_eq!(100, tx.iter(&tree).count()); + + let iter = tx.iter(&tree); + tree.insert("3", "3")?; + assert_eq!(100, iter.count()); + + Ok(()) +} diff --git a/crates/fjall/tests/write_buffer_size.rs b/crates/fjall/tests/write_buffer_size.rs new file mode 100644 index 000000000..507713f9a --- /dev/null +++ b/crates/fjall/tests/write_buffer_size.rs @@ -0,0 +1,58 @@ +use fjall::{Database, KeyspaceCreateOptions, KvSeparationOptions}; +use test_log::test; + +#[test] +fn write_buffer_size_after_insert() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + assert_eq!(0, db.write_buffer_size()); + + tree.insert("asd", "def")?; + + let write_buffer_size_after = db.write_buffer_size(); + assert!(write_buffer_size_after > 0); + + let mut batch = db.batch(); + batch.insert(&tree, "dsa", "qwe"); + batch.commit()?; + + let write_buffer_size_after_batch = db.write_buffer_size(); + assert!(write_buffer_size_after_batch > write_buffer_size_after); + + tree.rotate_memtable_and_wait()?; + assert_eq!(0, db.write_buffer_size()); + + Ok(()) +} + +#[test] +fn write_buffer_size_blob() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().with_kv_separation(Some(KvSeparationOptions::default())) + })?; + assert_eq!(0, db.write_buffer_size()); + + tree.insert("asd", "def")?; + + let write_buffer_size_after = db.write_buffer_size(); + assert!(write_buffer_size_after > 0); + + let mut batch = db.batch(); + batch.insert(&tree, "dsa", "qwe"); + batch.commit()?; + + let write_buffer_size_after_batch = db.write_buffer_size(); + assert!(write_buffer_size_after_batch > write_buffer_size_after); + + tree.rotate_memtable_and_wait()?; + assert_eq!(0, db.write_buffer_size()); + + Ok(()) +} diff --git a/crates/fjall/tests/write_during_read.rs b/crates/fjall/tests/write_during_read.rs new file mode 100644 index 000000000..45ccd0728 --- /dev/null +++ b/crates/fjall/tests/write_during_read.rs @@ -0,0 +1,25 @@ +use fjall::{Database, KeyspaceCreateOptions}; +use test_log::test; + +#[test] +fn write_during_read() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = Database::builder(&folder).open()?; + + let tree = db.keyspace("default", || { + KeyspaceCreateOptions::default().max_memtable_size(128_000) + })?; + + for x in 0u64..50_000 { + tree.insert(x.to_be_bytes(), x.to_be_bytes())?; + } + tree.rotate_memtable_and_wait()?; + + for guard in tree.iter() { + let (k, v) = guard.into_inner()?; + tree.insert(k, v.repeat(4))?; + } + + Ok(()) +} diff --git a/crates/fjall/tests/write_tx.rs b/crates/fjall/tests/write_tx.rs new file mode 100644 index 000000000..01fced988 --- /dev/null +++ b/crates/fjall/tests/write_tx.rs @@ -0,0 +1,20 @@ +use fjall::{KeyspaceCreateOptions, Readable, SingleWriterTxDatabase}; +use test_log::test; + +#[test] +fn write_tx_multi_keys() -> fjall::Result<()> { + let folder = tempfile::tempdir()?; + + let db = SingleWriterTxDatabase::builder(&folder).open()?; + let tree = db.keyspace("default", KeyspaceCreateOptions::default)?; + + let mut wtx = db.write_tx(); + wtx.insert(&tree, "1", "abc"); + wtx.insert(&tree, "1", "def"); + wtx.insert(&tree, "1", "ghi"); + assert_eq!(&*wtx.get(&tree, "1")?.unwrap(), b"ghi"); + wtx.commit()?; + assert_eq!(&*tree.get("1")?.unwrap(), b"ghi"); + + Ok(()) +} diff --git a/crates/lsm-tree/.config/nextest.toml b/crates/lsm-tree/.config/nextest.toml new file mode 100644 index 000000000..6079e36c6 --- /dev/null +++ b/crates/lsm-tree/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.default] +slow-timeout = { period = "10s", terminate-after = 6 } diff --git a/crates/lsm-tree/.gitignore b/crates/lsm-tree/.gitignore new file mode 100644 index 000000000..c31cd55da --- /dev/null +++ b/crates/lsm-tree/.gitignore @@ -0,0 +1,26 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +.directory + +.lsm.data +.test* +.bench + +mutants* +profile.json +fuzz*/**/out* + +old diff --git a/crates/lsm-tree/.rustfmt.toml b/crates/lsm-tree/.rustfmt.toml new file mode 100644 index 000000000..1c7741475 --- /dev/null +++ b/crates/lsm-tree/.rustfmt.toml @@ -0,0 +1,3 @@ +reorder_imports = true +# group_imports = "StdExternalCrate" +# imports_granularity = "crate" diff --git a/crates/lsm-tree/CONTRIBUTING.md b/crates/lsm-tree/CONTRIBUTING.md new file mode 100644 index 000000000..b5aab0e18 --- /dev/null +++ b/crates/lsm-tree/CONTRIBUTING.md @@ -0,0 +1,11 @@ +# Contributing + +## License + +By contributing to this project, you agree that your contributions will be licensed under the project's license (MIT OR Apache-2.0). + +Thank you for your contribution! + +## Looking for issues? + +https://github.com/fjall-rs/lsm-tree/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 diff --git a/crates/lsm-tree/Cargo.toml b/crates/lsm-tree/Cargo.toml new file mode 100644 index 000000000..b83a8315d --- /dev/null +++ b/crates/lsm-tree/Cargo.toml @@ -0,0 +1,97 @@ +[package] +name = "lsm-tree" +description = "A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs)" +license.workspace = true +version.workspace = true +edition.workspace = true +readme.workspace = true +include = ["src/**/*", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] +repository.workspace = true +homepage.workspace = true +keywords = ["lsmt", "lsm", "rocksdb", "leveldb", "key-value"] +categories = ["data-structures", "database-implementations", "algorithms"] + +[lib] +name = "lsm_tree" +path = "src/lib.rs" + +[features] +default = [] +lz4 = ["dep:lz4_flex"] +bytes_1 = ["dep:bytes"] +metrics = [] + +[dependencies] +bytes = { version = "1", optional = true } +byteorder = { package = "byteorder-lite", version = "0.1.0" } +byteview = { workspace = true } +crossbeam-skiplist = "0.1.3" +enum_dispatch = "0.3.13" +interval-heap = "0.0.5" +log = { workspace = true } +lz4_flex = { workspace = true, features = ["checked-decode"], optional = true } +quick_cache = { version = "0.6.16", default-features = false, features = [] } +rustc-hash = { workspace = true } +self_cell = "1.2.0" +sfa = "~1.0.0" +tempfile = { workspace = true } +varint-rs = "2.2.0" +xxhash-rust = { version = "0.8.15", features = ["xxh3"] } + +[dev-dependencies] +criterion = { version = "0.8.0", features = ["html_reports"] } +fs_extra = "1.3.0" +nanoid = "0.4.0" +rand = "0.9.2" +strum = { version = "0.28.0", features = ["derive"] } +test-log = "0.2.18" + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = [ + 'cfg(coverage,coverage_nightly)', +] } + +[package.metadata.cargo-all-features] +denylist = [] + +[[bench]] +name = "merge" +harness = false +path = "benches/merge.rs" +required-features = [] + +[[bench]] +name = "memtable" +harness = false +path = "benches/memtable.rs" +required-features = [] + +[[bench]] +name = "bloom" +harness = false +path = "benches/bloom.rs" +required-features = [] + +[[bench]] +name = "block" +harness = false +path = "benches/block.rs" +required-features = ["lz4"] + +[[bench]] +name = "tree" +harness = false +path = "benches/tree.rs" +required-features = ["lz4"] + +[[bench]] +name = "level_manifest" +harness = false +path = "benches/level_manifest.rs" +required-features = [] + +[[bench]] +name = "fd_table" +harness = false +path = "benches/fd_table.rs" +required-features = [] diff --git a/crates/lsm-tree/LICENSE-APACHE b/crates/lsm-tree/LICENSE-APACHE new file mode 100644 index 000000000..0e5254213 --- /dev/null +++ b/crates/lsm-tree/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/crates/lsm-tree/LICENSE-MIT b/crates/lsm-tree/LICENSE-MIT new file mode 100644 index 000000000..97f5e0faa --- /dev/null +++ b/crates/lsm-tree/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 fjall-rs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/lsm-tree/README.md b/crates/lsm-tree/README.md new file mode 100644 index 000000000..533fdac36 --- /dev/null +++ b/crates/lsm-tree/README.md @@ -0,0 +1,83 @@ +

+ +

+ +[![CI](https://github.com/fjall-rs/lsm-tree/actions/workflows/test.yml/badge.svg)](https://github.com/fjall-rs/lsm-tree/actions/workflows/test.yml) +[![docs.rs](https://img.shields.io/docsrs/lsm-tree?color=green)](https://docs.rs/lsm-tree) +[![Crates.io](https://img.shields.io/crates/v/lsm-tree?color=blue)](https://crates.io/crates/lsm-tree) +![MSRV](https://img.shields.io/badge/MSRV-1.90.0-blue) +[![dependency status](https://deps.rs/repo/github/fjall-rs/lsm-tree/status.svg)](https://deps.rs/repo/github/fjall-rs/lsm-tree) + +A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs) in Rust. + +> [!NOTE] +> This crate only provides a primitive LSM-tree, not a full storage engine. +> For example, it does not ship with a write-ahead log. +> You probably want to use https://github.com/fjall-rs/fjall instead. + +## About + +This is the most feature-rich LSM-tree implementation in Rust! It features: + +- Thread-safe `BTreeMap`-like API +- Mostly [safe](./UNSAFE.md) & 100% stable Rust +- Block-based tables with compression support & prefix truncation + - Optional block hash indexes in data blocks for faster point lookups [[3]](#footnotes) + - Per-level filter/index block pinning configuration +- Range & prefix searching with forward and reverse iteration +- Block caching to keep hot data in memory +- File descriptor caching with upper bound to reduce `fopen` syscalls +- *AMQ* filters (currently Bloom filters) to improve point lookup performance +- Multi-versioning of KVs, enabling snapshot reads +- Optionally partitioned block index & filters for better cache efficiency [[1]](#footnotes) +- Leveled and FIFO compaction +- Optional key-value separation for large value workloads [[2]](#footnotes), with automatic garbage collection +- Single deletion tombstones ("weak" deletion) +- Optional compaction filters to run custom logic during compactions + +Keys are limited to 65536 bytes, values are limited to 2^32 bytes. +As is normal with any kind of storage engine, larger keys and values have a bigger performance impact. + +## Sponsors + + + + + + Orbitinghail + + + +## Feature flags + +### lz4 + +Allows using `LZ4` compression, powered by [`lz4_flex`](https://github.com/PSeitz/lz4_flex). + +*Disabled by default.* + +### bytes_1 + +Uses [`bytes`](https://github.com/tokio-rs/bytes) 1.x as the underlying `Slice` type. + +*Disabled by default.* + +## Run unit benchmarks + +```bash +cargo bench --features lz4 +``` + +## License + +All source code is licensed under MIT OR Apache-2.0. + +All contributions are to be licensed as MIT OR Apache-2.0. + +## Footnotes + +[1] https://rocksdb.org/blog/2017/05/12/partitioned-index-filter.html + +[2] https://github.com/facebook/rocksdb/wiki/BlobDB + +[3] https://rocksdb.org/blog/2018/08/23/data-block-hash-index.html diff --git a/crates/lsm-tree/UNSAFE.md b/crates/lsm-tree/UNSAFE.md new file mode 100644 index 000000000..facaef443 --- /dev/null +++ b/crates/lsm-tree/UNSAFE.md @@ -0,0 +1,33 @@ +# Unsafe usage + +... + +## Run fuzz testing + +```bash +cd fuzz/data_block +mkdir in +cat /dev/random | head -n 100 > in/input +cargo afl build && cargo afl fuzz -i in -o out target/debug/data_block + +cd fuzz/index_block +mkdir in +cat /dev/random | head -n 100 > in/input +cargo afl build && cargo afl fuzz -i in -o out target/debug/index_block + +cd fuzz/table_read +mkdir in +cat /dev/random | head -n 100 > in/input +cargo afl build && cargo afl fuzz -i in -o out target/debug/table_read + +cd fuzz/compare_prefixed_slice +mkdir in +cat /dev/random | head -n 100 > in/input +cargo afl build && cargo afl fuzz -i in -o out target/debug/compare_prefixed_slice +``` + +## Run mutation testing + +```bash +cargo-mutants mutants +``` diff --git a/crates/lsm-tree/benches/block.rs b/crates/lsm-tree/benches/block.rs new file mode 100644 index 000000000..f0ae8beb7 --- /dev/null +++ b/crates/lsm-tree/benches/block.rs @@ -0,0 +1,62 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use lsm_tree::{ + table::block::{Block, BlockType}, + CompressionType, +}; + +fn input(size: usize) -> Vec { + (0..size) + .map(|index| { + let index = index as u64; + index + .wrapping_mul(6_364_136_223_846_793_005) + .rotate_left(17) as u8 + }) + .collect() +} + +fn encode_block(c: &mut Criterion) { + let mut group = c.benchmark_group("Encode block"); + + for compression in [CompressionType::None, CompressionType::Lz4] { + for kib in [4, 8, 16, 32, 64, 128] { + let data = input(kib * 1_024); + group.throughput(Throughput::Bytes(data.len() as u64)); + + group.bench_function(format!("{kib} KiB [{compression}]"), |b| { + b.iter_batched( + || Vec::with_capacity(data.len()), + |mut encoded| { + Block::write_into(&mut encoded, &data, BlockType::Data, compression) + .unwrap(); + encoded + }, + BatchSize::SmallInput, + ); + }); + } + } +} + +fn decode_block(c: &mut Criterion) { + let mut group = c.benchmark_group("Decode block"); + + for compression in [CompressionType::None, CompressionType::Lz4] { + for kib in [4, 8, 16, 32, 64, 128] { + let data = input(kib * 1_024); + let mut encoded = Vec::with_capacity(data.len()); + Block::write_into(&mut encoded, &data, BlockType::Data, compression).unwrap(); + + group.throughput(Throughput::Bytes(data.len() as u64)); + group.bench_function(format!("{kib} KiB [{compression}]"), |b| { + b.iter(|| { + let block = Block::from_reader(&mut &encoded[..], compression).unwrap(); + assert_eq!(block.data.len(), data.len()); + }); + }); + } + } +} + +criterion_group!(benches, encode_block, decode_block); +criterion_main!(benches); diff --git a/crates/lsm-tree/benches/bloom.rs b/crates/lsm-tree/benches/bloom.rs new file mode 100644 index 000000000..4c2d8dfe1 --- /dev/null +++ b/crates/lsm-tree/benches/bloom.rs @@ -0,0 +1,107 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use rand::{Rng, RngCore}; + +// Not really worth it anymore on new CPUs...? +fn fast_block_index(c: &mut Criterion) { + pub fn fast_impl(h: u64, num_blocks: usize) -> usize { + // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + (((h >> 32).wrapping_mul(num_blocks as u64)) >> 32) as usize + } + + let mut rng = rand::rng(); + let num_blocks = 100_000; + + c.bench_function("block index - mod", |b| { + b.iter(|| { + let h: u64 = rng.random(); + std::hint::black_box(h % (num_blocks as u64)) + }); + }); + + c.bench_function("block index - fast", |b| { + b.iter(|| { + let h: u64 = rng.random(); + std::hint::black_box(fast_impl(h, num_blocks)) + }); + }); +} + +fn standard_filter_construction(c: &mut Criterion) { + use lsm_tree::table::filter::standard_bloom::Builder; + + let mut rng = rand::rng(); + + c.bench_function("standard bloom filter add key, 1M", |b| { + let mut filter = Builder::with_fp_rate(1_000_000, 0.01); + + b.iter(|| { + let mut key = [0; 16]; + rng.fill_bytes(&mut key); + + filter.set_with_hash(Builder::get_hash(&key)); + }); + }); + + c.bench_function("standard bloom filter add key, 10M", |b| { + let mut filter = Builder::with_fp_rate(10_000_000, 0.01); + + b.iter(|| { + let mut key = [0; 16]; + rng.fill_bytes(&mut key); + + filter.set_with_hash(Builder::get_hash(&key)); + }); + }); +} + +fn standard_filter_contains(c: &mut Criterion) { + use lsm_tree::table::filter::standard_bloom::Builder; + + let keys = (0..100_000u128) + .map(|x| x.to_be_bytes().to_vec()) + .collect::>(); + + for fpr in [0.1, 0.01, 0.001, 0.0001, 0.00001] { + // NOTE: Purposefully bloat bloom filter size to run into more CPU cache misses + let n = 100_000_000; + + let mut filter = Builder::with_fp_rate(n, fpr); + + for key in &keys { + filter.set_with_hash(Builder::get_hash(key)); + } + + let mut rng = rand::rng(); + + let filter_bytes = filter.build(); + + c.bench_function( + &format!( + "standard bloom filter contains key, true positive ({}%)", + fpr * 100.0, + ), + |b| { + b.iter(|| { + use rand::seq::IndexedRandom; + use lsm_tree::table::filter::standard_bloom::StandardBloomFilterReader as Reader; + + // NOTE: To make the costs more realistic, we + // pretend we are reading the filter straight from the block + let filter = Reader::new(&filter_bytes).unwrap(); + + let sample = keys.choose(&mut rng).unwrap(); + let hash = Builder::get_hash(sample); + assert!(filter.contains_hash(hash)); + }); + }, + ); + } +} + +criterion_group!( + benches, + fast_block_index, + standard_filter_construction, + standard_filter_contains, +); +criterion_main!(benches); diff --git a/crates/lsm-tree/benches/fd_table.rs b/crates/lsm-tree/benches/fd_table.rs new file mode 100644 index 000000000..79632fa1f --- /dev/null +++ b/crates/lsm-tree/benches/fd_table.rs @@ -0,0 +1,101 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use std::sync::Arc; + +fn file_descriptor_table(c: &mut Criterion) { + use std::fs::File; + + let file = tempfile::NamedTempFile::new().unwrap(); + + let mut group = c.benchmark_group("Get file descriptor"); + + let id = (0, 523).into(); + let descriptor_table = lsm_tree::descriptor_table::DescriptorTable::new(100); + descriptor_table.insert_for_table(id, Arc::new(file.into_file())); + + group.bench_function("descriptor table", |b: &mut criterion::Bencher<'_>| { + b.iter(|| { + let _guard = descriptor_table.access_for_table(&id).unwrap(); + }); + }); + + let file = tempfile::NamedTempFile::new().unwrap(); + + group.bench_function("fopen", |b: &mut criterion::Bencher<'_>| { + b.iter(|| { + File::open(file.path()).unwrap(); + }); + }); +} + +fn file_descriptor_table_threading(c: &mut Criterion) { + use std::fs::File; + + let files_to_open = 1_000; + + let file = tempfile::NamedTempFile::new().unwrap(); + let file = Arc::new(file.into_file()); + + let mut group = c.benchmark_group("Get file descriptor (threaded)"); + group.throughput(criterion::Throughput::Elements(files_to_open as u64)); + + for thread_count in [1, 2, 4, 8, 16] { + let id = (0, 523).into(); + let descriptor_table = Arc::new(lsm_tree::descriptor_table::DescriptorTable::new(100)); + descriptor_table.insert_for_table(id, file.clone()); + + group.bench_function( + format!("descriptor table - {thread_count} threads"), + |b: &mut criterion::Bencher<'_>| { + b.iter(|| { + let threads = (0..thread_count) + .map(|_| { + let table = descriptor_table.clone(); + + std::thread::spawn(move || { + for _ in 0..(files_to_open / thread_count) { + let _guard = table.access_for_table(&id).unwrap(); + } + }) + }) + .collect::>(); + + for thread in threads { + thread.join().unwrap(); + } + }); + }, + ); + + let file = Box::leak(Box::new(tempfile::NamedTempFile::new().unwrap())); + + group.bench_function( + format!("fopen - {thread_count} threads"), + |b: &mut criterion::Bencher<'_>| { + b.iter(|| { + let threads = (0..thread_count) + .map(|_| { + let path = file.path(); + + std::thread::spawn(move || { + for _ in 0..(files_to_open / thread_count) { + File::open(path).unwrap(); + } + }) + }) + .collect::>(); + + for thread in threads { + thread.join().unwrap(); + } + }); + }, + ); + } +} + +criterion_group!( + benches, + file_descriptor_table, + file_descriptor_table_threading, +); +criterion_main!(benches); diff --git a/crates/lsm-tree/benches/level_manifest.rs b/crates/lsm-tree/benches/level_manifest.rs new file mode 100644 index 000000000..c260a58b9 --- /dev/null +++ b/crates/lsm-tree/benches/level_manifest.rs @@ -0,0 +1,33 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use lsm_tree::{config::BlockSizePolicy, AbstractTree, Config}; + +fn iterate_tables(c: &mut Criterion) { + let mut group = c.benchmark_group("Iterate version tables"); + group.sample_size(10); + + std::fs::create_dir_all(".bench").unwrap(); + + for segment_count in [0, 1, 5, 10, 100, 500, 1_000, 2_000, 4_000] { + group.bench_function(format!("iterate {segment_count} segments"), |b| { + let folder = tempfile::tempdir_in(".bench").unwrap(); + let tree = Config::new(folder, Default::default(), Default::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .open() + .unwrap(); + + for x in 0_u64..segment_count { + tree.insert("a", "b", x); + tree.flush_active_memtable(0).unwrap(); + } + + let version = tree.current_version(); + + b.iter(|| { + assert_eq!(version.iter_tables().count(), segment_count as usize); + }); + }); + } +} + +criterion_group!(benches, iterate_tables); +criterion_main!(benches); diff --git a/crates/lsm-tree/benches/memtable.rs b/crates/lsm-tree/benches/memtable.rs new file mode 100644 index 000000000..db708ddfb --- /dev/null +++ b/crates/lsm-tree/benches/memtable.rs @@ -0,0 +1,115 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use lsm_tree::{InternalValue, Memtable, SeqNo}; +use nanoid::nanoid; + +fn memtable_get_hit(c: &mut Criterion) { + let memtable = Memtable::new(0); + + memtable.insert(InternalValue::from_components( + "abc_w5wa35aw35naw", + vec![1, 2, 3], + 0, + lsm_tree::ValueType::Value, + )); + + for _ in 0..1_000_000 { + memtable.insert(InternalValue::from_components( + format!("abc_{}", nanoid!()).as_bytes(), + vec![], + 0, + lsm_tree::ValueType::Value, + )); + } + + c.bench_function("memtable get", |b| { + b.iter(|| { + assert_eq!( + [1, 2, 3], + &*memtable + .get(b"abc_w5wa35aw35naw", SeqNo::MAX) + .unwrap() + .value, + ) + }); + }); +} + +fn memtable_get_snapshot(c: &mut Criterion) { + let memtable = Memtable::new(0); + + memtable.insert(InternalValue::from_components( + "abc_w5wa35aw35naw", + vec![1, 2, 3], + 0, + lsm_tree::ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + "abc_w5wa35aw35naw", + vec![1, 2, 3, 4], + 1, + lsm_tree::ValueType::Value, + )); + + for _ in 0..1_000_000 { + memtable.insert(InternalValue::from_components( + format!("abc_{}", nanoid!()).as_bytes(), + vec![], + 0, + lsm_tree::ValueType::Value, + )); + } + + c.bench_function("memtable get snapshot", |b| { + b.iter(|| { + assert_eq!( + [1, 2, 3], + &*memtable.get(b"abc_w5wa35aw35naw", 1).unwrap().value, + ); + }); + }); +} + +fn memtable_get_miss(c: &mut Criterion) { + let memtable = Memtable::new(0); + + for _ in 0..1_000_000 { + memtable.insert(InternalValue::from_components( + format!("abc_{}", nanoid!()).as_bytes(), + vec![], + 0, + lsm_tree::ValueType::Value, + )); + } + + c.bench_function("memtable get miss", |b| { + b.iter(|| assert!(memtable.get(b"abc_564321", SeqNo::MAX).is_none())); + }); +} + +fn memtable_highest_seqno(c: &mut Criterion) { + c.bench_function("memtable highest seqno", |b| { + let memtable = Memtable::new(0); + + for x in 0..100_000 { + memtable.insert(InternalValue::from_components( + format!("abc_{}", nanoid!()).as_bytes(), + vec![], + x, + lsm_tree::ValueType::Value, + )); + } + + b.iter(|| { + assert_eq!(Some(99_999), memtable.get_highest_seqno()); + }); + }); +} + +criterion_group!( + benches, + memtable_get_hit, + memtable_get_snapshot, + memtable_get_miss, + memtable_highest_seqno +); +criterion_main!(benches); diff --git a/crates/lsm-tree/benches/merge.rs b/crates/lsm-tree/benches/merge.rs new file mode 100644 index 000000000..cb5f24024 --- /dev/null +++ b/crates/lsm-tree/benches/merge.rs @@ -0,0 +1,77 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use lsm_tree::merge::{BoxedIterator, Merger}; +use lsm_tree::{mvcc_stream::MvccStream, InternalValue, Memtable}; +use nanoid::nanoid; + +fn merger(c: &mut Criterion) { + for num in [2, 4, 8, 16, 30] { + c.bench_function(&format!("Merge {num}"), |b| { + let memtables = (0..num) + .map(|id| { + let table = Memtable::new(id as u64); + + for _ in 0..100 { + table.insert(InternalValue::from_components( + nanoid!(), + vec![], + 0, + lsm_tree::ValueType::Value, + )); + } + + table + }) + .collect::>(); + + b.iter_with_large_drop(|| { + let iters = memtables + .iter() + .map(|x| x.iter().map(Ok)) + .map(|x| Box::new(x) as BoxedIterator<'_>) + .collect(); + + let merger = Merger::new(iters); + + assert_eq!(num * 100, merger.count()); + }) + }); + } +} + +fn mvcc_stream(c: &mut Criterion) { + for num in [2, 4, 8, 16, 30] { + c.bench_function(&format!("MVCC stream {num} versions"), |b| { + let memtables = (0..num) + .map(|id| { + let table = Memtable::new(id as u64); + + for key in 'a'..='z' { + table.insert(InternalValue::from_components( + key.to_string(), + vec![], + num, + lsm_tree::ValueType::Value, + )); + } + + table + }) + .collect::>(); + + b.iter_with_large_drop(|| { + let iters = memtables + .iter() + .map(|x| x.iter().map(Ok)) + .map(|x| Box::new(x) as BoxedIterator<'_>) + .collect(); + + let merger = MvccStream::new(Merger::new(iters)); + + assert_eq!(26, merger.count()); + }) + }); + } +} + +criterion_group!(benches, merger, mvcc_stream); +criterion_main!(benches); diff --git a/crates/lsm-tree/benches/tree.rs b/crates/lsm-tree/benches/tree.rs new file mode 100644 index 000000000..6e457b93d --- /dev/null +++ b/crates/lsm-tree/benches/tree.rs @@ -0,0 +1,412 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use lsm_tree::{ + config::BlockSizePolicy, AbstractTree, Cache, Config, Guard, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use std::{path::Path, sync::Arc}; +use tempfile::tempdir; + +fn config(path: impl AsRef) -> Config { + Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) +} + +fn full_scan(c: &mut Criterion) { + let mut group = c.benchmark_group("scan all"); + group.sample_size(10); + + for item_count in [10_000, 100_000, 1_000_000] { + group.bench_function(format!("scan all uncached, {item_count} items"), |b| { + let path = tempdir().unwrap(); + + let tree = config(path) + .use_cache(Cache::with_capacity_bytes(0).into()) + .open() + .unwrap(); + + for x in 0_u32..item_count { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value, 0); + } + + tree.flush_active_memtable(0).unwrap(); + + b.iter(|| { + assert_eq!(tree.len(SeqNo::MAX, None).unwrap(), item_count as usize); + }) + }); + + group.bench_function(format!("scan all cached, {item_count} items"), |b| { + let path = tempdir().unwrap(); + + let tree = config(path) + .use_cache(Cache::with_capacity_bytes(100_000_000).into()) + .open() + .unwrap(); + + for x in 0_u32..item_count { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value, 0); + } + + tree.flush_active_memtable(0).unwrap(); + assert_eq!(tree.len(SeqNo::MAX, None).unwrap(), item_count as usize); + + b.iter(|| { + assert_eq!(tree.len(SeqNo::MAX, None).unwrap(), item_count as usize); + }) + }); + } +} + +fn scan_vs_query(c: &mut Criterion) { + use std::ops::Bound::*; + + let mut group = c.benchmark_group("scan vs query"); + + for size in [100_000, 1_000_000] { + let path = tempdir().unwrap(); + + let tree = config(path) + .use_cache(Cache::with_capacity_bytes(0).into()) + .open() + .unwrap(); + + for x in 0..size as u64 { + let key = x.to_be_bytes().to_vec(); + let value = nanoid::nanoid!().as_bytes().to_vec(); + tree.insert(key, value, 0); + } + + tree.flush_active_memtable(0).unwrap(); + assert_eq!(tree.len(SeqNo::MAX, None).unwrap(), size); + + group.sample_size(10); + group.bench_function(format!("scan {} (uncached)", size), |b| { + b.iter(|| { + let count = tree + .iter(SeqNo::MAX, None) + .filter_map(|item| item.key().ok()) + .filter(|key| { + let buf = &key[..8]; + let (int_bytes, _rest) = buf.split_at(std::mem::size_of::()); + let num = u64::from_be_bytes(int_bytes.try_into().unwrap()); + (60000..60010).contains(&num) + }) + .count(); + assert_eq!(count, 10); + }) + }); + group.bench_function(format!("query {} (uncached)", size), |b| { + b.iter(|| { + let iter = tree.range( + ( + Included(60000_u64.to_be_bytes().to_vec()), + Excluded(60010_u64.to_be_bytes().to_vec()), + ), + SeqNo::MAX, + None, + ); + let iter = iter.into_iter(); + assert_eq!(iter.count(), 10); + }) + }); + group.bench_function(format!("query rev {}", size), |b| { + b.iter(|| { + let iter = tree.range( + ( + Included(60000_u64.to_be_bytes().to_vec()), + Excluded(60010_u64.to_be_bytes().to_vec()), + ), + SeqNo::MAX, + None, + ); + let iter = iter.into_iter(); + assert_eq!(iter.rev().count(), 10); + }) + }); + } +} + +fn scan_vs_prefix(c: &mut Criterion) { + let mut group = c.benchmark_group("scan vs prefix"); + + for size in [10_000, 100_000, 1_000_000] { + let path = tempdir().unwrap(); + + let tree = config(path) + .use_cache(Cache::with_capacity_bytes(0).into()) + .open() + .unwrap(); + + for _ in 0..size { + let key = nanoid::nanoid!(); + let value = nanoid::nanoid!(); + tree.insert(key, value, 0); + } + + let prefix = "hello$$$"; + + for _ in 0..10_u64 { + let key = format!("{}:{}", prefix, nanoid::nanoid!()); + let value = nanoid::nanoid!(); + tree.insert(key, value, 0); + } + + tree.flush_active_memtable(0).unwrap(); + assert_eq!(tree.len(SeqNo::MAX, None).unwrap() as u64, size + 10); + + group.sample_size(10); + group.bench_function(format!("scan {} (uncached)", size), |b| { + b.iter(|| { + let iter = tree + .iter(SeqNo::MAX, None) + .filter_map(|item| item.key().ok()) + .filter(|key| key.starts_with(prefix.as_bytes())); + assert_eq!(iter.count(), 10); + }); + }); + group.bench_function(format!("prefix {} (uncached)", size), |b| { + b.iter(|| { + let iter = tree.prefix(prefix, SeqNo::MAX, None); + assert_eq!(iter.count(), 10); + }); + }); + group.bench_function(format!("prefix rev {} (uncached)", size), |b| { + b.iter(|| { + let iter = tree.prefix(prefix, SeqNo::MAX, None); + assert_eq!(iter.rev().count(), 10); + }); + }); + } +} + +fn tree_get_pairs(c: &mut Criterion) { + let mut group = c.benchmark_group("Get pairs"); + group.sample_size(10); + + for segment_count in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] { + { + let folder = tempfile::tempdir().unwrap(); + let tree = config(folder) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .use_cache(Arc::new(Cache::with_capacity_bytes(0))) + .open() + .unwrap(); + + let mut x = 0_u64; + + for _ in 0..segment_count { + for _ in 0..10 { + let key = x.to_be_bytes(); + x += 1; + tree.insert(key, key, 0); + } + tree.flush_active_memtable(0).unwrap(); + } + + group.bench_function( + &format!("Tree::first_key_value (disjoint), {segment_count} segments"), + |b| { + b.iter(|| { + assert!(tree.first_key_value(SeqNo::MAX, None).is_some()); + }); + }, + ); + + group.bench_function( + &format!("Tree::last_key_value (disjoint), {segment_count} segments"), + |b| { + b.iter(|| { + assert!(tree.last_key_value(SeqNo::MAX, None).is_some()); + }); + }, + ); + } + + { + let folder = tempfile::tempdir().unwrap(); + let tree = config(folder) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .use_cache(Arc::new(Cache::with_capacity_bytes(0))) + .open() + .unwrap(); + + let mut x = 0_u64; + + for _ in 0..segment_count { + for _ in 0..10 { + let key = x.to_be_bytes(); + x += 1; + tree.insert(key, key, 0); + } + tree.insert("a", vec![], 0); + tree.insert(u64::MAX.to_be_bytes(), vec![], 0); + tree.flush_active_memtable(0).unwrap(); + } + + group.bench_function( + &format!("Tree::first_key_value (non-disjoint), {segment_count} segments"), + |b| { + b.iter(|| { + assert!(tree.first_key_value(SeqNo::MAX, None).is_some()); + }); + }, + ); + + group.bench_function( + &format!("Tree::last_key_value (non-disjoint), {segment_count} segments"), + |b| { + b.iter(|| { + assert!(tree.last_key_value(SeqNo::MAX, None).is_some()); + }); + }, + ); + } + } +} + +fn disk_point_read(c: &mut Criterion) { + let folder = tempdir().unwrap(); + + let tree = config(folder) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .use_cache(Arc::new(Cache::with_capacity_bytes(0))) + .open() + .unwrap(); + + for seqno in 0..5 { + tree.insert("a", "b", seqno); + } + tree.flush_active_memtable(0).unwrap(); + + for seqno in 5..10 { + tree.insert("a", "b", seqno); + } + tree.flush_active_memtable(0).unwrap(); + + c.bench_function("point read latest (uncached)", |b| { + let tree = tree.clone(); + + b.iter(|| { + tree.get("a", SeqNo::MAX).unwrap().unwrap(); + }); + }); + + c.bench_function("point read at seqno (uncached)", |b| { + b.iter(|| { + tree.get("a", 5).unwrap().unwrap(); + }); + }); +} + +fn disjoint_tree_minmax(c: &mut Criterion) { + let mut group = c.benchmark_group("Disjoint tree"); + + let folder = tempfile::tempdir().unwrap(); + + let tree = config(folder) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .use_cache(Arc::new(Cache::with_capacity_bytes(0))) + .open() + .unwrap(); + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0).unwrap(); + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 6)), 0) + .unwrap(); + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0).unwrap(); + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 5)), 0) + .unwrap(); + + tree.insert("c", "c", 0); + tree.flush_active_memtable(0).unwrap(); + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 4)), 0) + .unwrap(); + + tree.insert("d", "d", 0); + tree.flush_active_memtable(0).unwrap(); + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 3)), 0) + .unwrap(); + + tree.insert("e", "e", 0); + tree.flush_active_memtable(0).unwrap(); + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 2)), 0) + .unwrap(); + + tree.insert("f", "f", 0); + tree.flush_active_memtable(0).unwrap(); + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 1)), 0) + .unwrap(); + + tree.insert("g", "g", 0); + tree.flush_active_memtable(0).unwrap(); + + group.bench_function("Tree::first_key_value".to_string(), |b| { + b.iter(|| { + assert_eq!( + &*tree + .first_key_value(SeqNo::MAX, None) + .unwrap() + .key() + .unwrap(), + b"a" + ); + }); + }); + + group.bench_function("Tree::last_key_value".to_string(), |b| { + b.iter(|| { + assert_eq!( + &*tree + .last_key_value(SeqNo::MAX, None) + .unwrap() + .key() + .unwrap(), + b"g" + ); + }); + }); +} + +fn blob_tree_get(c: &mut Criterion) { + let folder = tempfile::tempdir().unwrap(); + + let tree = config(folder.path()) + .use_cache(Cache::with_capacity_bytes(0).into()) + .with_kv_separation(Some(KvSeparationOptions::default())) + .open() + .unwrap(); + + let value = b"powek5bowa".repeat(100); + + tree.insert("mykey", &value, 0); + + c.bench_function("blob tree get", |b| { + b.iter(|| { + tree.get("mykey", SeqNo::MAX).unwrap().unwrap(); + }); + }); +} + +// TODO: benchmark point read disjoint vs non-disjoint level vs disjoint *tree* +// TODO: benchmark .prefix().next() and .next_back(), disjoint and non-disjoint + +criterion_group!( + benches, + blob_tree_get, + disjoint_tree_minmax, + disk_point_read, + full_scan, + scan_vs_query, + scan_vs_prefix, + tree_get_pairs, +); +criterion_main!(benches); diff --git a/crates/lsm-tree/logo.png b/crates/lsm-tree/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..d5873dd9aa3a365cb76ba6d3f19527a429bbb9e2 GIT binary patch literal 19998 zcmcG0^;=Zk7w;KT2|+{3zR!TY~1cVtxMMb3<8kFv?AuSp~IwT~d z8DfTk8D{40dB690?q6_!5apbG_TH;MYwfjt_u#%JGsAfX2!fckwQd_g5Eb|*6?B3Q z{DpZwf(L)Wo*8JW!zQ~W;oz5(URq|)Ac*k{NjnEYdf&Bg-!gojwl#Ay&2Tgw zN47D-nzvM<-xp+k5xgWW*wvaXKt)wbWhNTuo-yt=&?hu77o&ZxSGCt(w%h*dl`oD} zDL;f=7Tzc-R)q@;s-c6&Ta{M+crc0U${6`=w_pQTj2?{+&S1IEY`hp~SvmfsS!MZ^ zS^^&wGTXhd%Y@ANU^p9<633_sfpI4?V^m8)z=@E^D(b`8!Guc z2pQwod%J=iM0FN@FnoH3ZK-H2U$kYzTX!$ntW=v+4S zw+mQ>EWY4TyNL5=Sw!!Mzr=Yyy=q~OZ51pD&-IjB zqcL69`EwV&+@PIu8^1D>=7oHSi9~WSHp4)K}lKTOsiN}dBJZ?u=oijPgGW!*KKIkbR?fHbUVxb#TCZX1z{Oa(BsrBO-n^oqJBfa`3 z({T{=E3cj{RP=Nd`Ppiis77AojW~BI1o^9xh<23IXYthwpC1D*Tb(LAjk})FXfQZ#n|IS2AHBAr`!idAeph=MeKAE?|OA*+MLFhA~$ZcM6^5{29i>Cm9hbM5!qt-zU+XKiX?? zbvG1(6k;_qu)?vpTH;)1@etYA`JYYO`V;K+sFS=z2wMH{QLsChYJ*QT@t3@|D^LO% z{_G4slc?h1BNYg0_+ZUr%4b^t7>=41aQ}}xruu|=xDjfW8iF{7FU$*x)8ynjKQO0K z>zpq~AF*h6qUj+4-Bn(y{h`xT27J&>X@aU)-!Tl*dR_D9l$JA4iX9yN-wdF(g*obK zKHB1(ba|4HuN#(-5OhP?s0}!VW+yv!rTU05<+1qpHi@xNR_gKdkXEPB-onDe5D1!= zRN8YBB=mZt7p@)w=96s~nq&FX_Ve?{zfW*Bnii z)6MYjc2>l%()17+?=!Qc>3NjhF9SjUjEXxk?I`A^racWS28aZgk%`H9@u$Hs>x*Ixj&1)G$2U8og`G@@qI;c|6@7>O zn`B2U=1UN}eZghGCKbF_Rw2-oS*3lJ0S<`INPp$ebr8#VoHW#xiZwR)g3GTA5~vX8 zk0T25``5213?~4N)D#*Ckq!afpE5|{EOZFxq1mJI;Z7RBl4;$d&RC!WZ8wPY{7jII zu|7Fz4ftiXsgQ@X2=X(0B#ts0ePMq7$l}E_MwHotva?0i5=aYu&}AECW%m+Eu(A6O zu}Qq9^?&aWekdX{S*{coS*Q95rq0RTmb`AtWOVxs6hlW4DxD|H-s9NeW8NW91yT1W)~x96)k28 zA5be7me4Z+#LEg@$hgHPN3bceCQtcZ0ULW1TOSNIL<-dIk-vuov&ssI?G4}=+T}+# z29l)#l(6rs_k#wVsAXiWP6E?S%J+qy-)U(W^%Vi1+g)FWDl}To*`e3~Pp&BXK?)_- zA1?oIBm*y$G8)M?oB%|2g^-zK49pN7<2dL4k>9w~Tb9HGOKT@Whbhw=GCh2)y{EYePcIJui6PJbR$} zAp5RsNa^jDuKiz&;zEe!;<2d+)Fjjs>}Gc~_oa4syo z{@}Dh7??>R|79%OE)uJffg27|v3^Q+l{Tx~Vde|-CU&FHluh5Fe} zWSoQukq|i^u>BLh!WMZqCl~czBb)~TUOvyPmLsh()Y09fhC$E{pYq0dH9X{O*7V3` zTW#wn!Kmi^{A%(rCQ)sMA7T9U;0}0XiutUXOHKD)MqWmwP2UFg)U{w`)i#x~ zprr9)W-HN;O=}W8;tDIt4u7l&IERp1wfG3%V>H0KI-hm3p*&eyTGrJj!^YeZhQmZy zN|8k4S=+1NMVG(JAP$+ReM{SUYF6;im`f==A_RzCpm*&KJU8e7jm^V`_(FrXpaV59 zjU=+D6wa9#pVy=Fu5;r2Q+mo2Pqy6Q^l!&kdbA8hy2~8w=D}pWw;S2vuEnKJ zNw#+JeY!VwGN6<}R*>=q^?uCll1>o~v+M#e;+GsXpi#N6?1N zWc#hW-e|Q)*{{qkO3u*4n1MZ(UGw|*gzMV#2Ddr<`^~xP1lAqcofd*t=)=l2*^(Iu zziK;1ZO;e%*~aq`cwz$FoPjY?xP&gzJ=G?WNB{`4_j+?Hj~Mqg1u1uUlSby5tN})W z90v}{7nt=?6AmfjM6gZglPiJCxpTe0g$T^X4*ayaxeGG=n`3Q^?`Wn+9T0c@~bX=z)dZq~_yq=%tA!dL|QG z9Esn0p5s)6{V+E?MhLwbb34)h%P>9Z@ai9+h!`ovzjg6f6imlj}(ifEG8F|qGIW_d`oHpqi;ffp}D1Ww1 zZ9YWi=M?Qde^XQy@d6%m7>ujzUub)F^18ekOAaR>xcFXrrPgbNgv?>c@29sj0`Vsc zi%a{|j1IEU#Fc-XNz#iiF`I3rm0!)zy~O~-^!nJ`{KBkmeKmZrA|j`KXb3ee!x)Z{rFYmP4DKB1PXJNnOSZR-{F*#hrgWycC(-oA~IlO zkNP0}j)r>3*hUkMxc1-tR~9e!G;m)GI9gfXC^9;6m_35R_#4y7(}%73q!Jysa@PGJLP z@D8}Hvt>LXEu+Np{K!8p#roZm1FE^-5Q=r5mpM6gAC{}+^{!TWSrpgEjigF{H=;=F z7MeJ&3>kdR!{;=&?&^^_I<4)fhY|t)Bxu2ys`19%4;{oz&sI^ zTsC{Yp#H>q>C5}MCd7?VvPaClAGwWrQz#Kc25`DwN+D!8i5K^F069hY* z!I_nCz%Ov))YIePvq3waWo7oRwhTZG6lNQC;~e@X-Z>x5+;3rvjzh`-uahDzB&F^y z#rE{+Wve%ffOiTh8Q6iQ7EAmAVP_3q=(E1`F!1OoUznXCZ%f%gZ;mh^A$iT}Zi;Yj zZJn{;DLnf7CqN)iX-*DO(z4lnZ|FrKAw9eaDDr@=f*bjkrVL4f``V8y|C!bK33D6W z;(0!b9@P9reuyo+itg(Q=2Z=PU>!uNXRAL7#e9o35c7Sym|6CgCVVJQ9om4QA^^)P zF-*9QlUJ9Diaom~g87uIY_7Lrg5OR1{wtg&9-LPi0iwG4+B{Wca5bYtFZnj$r73D- z4l}8n>)F;P?LS?+uQ%DceAdGHq?hhu<}aO#XSOV;g-#FWDo~WbZmpGkbw!%f^9@Vm zZ)6${J$Z4u-Xq>iA^tZ`gur3P93ecP+8z-ttQAPZ}mQ#!ql;(%KEN`-FmJ- zPhKkEq$Dk_w_JL6`u+cm0#klvmw(u4FJ&-?qfjP`s<3UkE3;e`Fp@hOJVZissM^`Jc3W zlMk;E3#~-q@Gvv0tDg##MAeQ|BS$3Mthb=+8!fO$6WmGIm=^q-oOXe-cqAB58*5ELE?4_++A z`sMXjNC7T&n)!}v=9+j0qXWiEU)lsO>+fylZ8Io_zAypbmV+Q{Di!xQ;}=- zvR^MxqEdvy+isal@lCy|LVQug`li?Fu)vL4)!x%*sfKQq1^FEF%`#aYoa-=W#SHPk64aT`dCuMVdOfYXv4uf%%kA?%u!u zNzww#)!qASRvB$up=TZXx85Ld&r~Or|05iADLH=ZO_1wjfho%14n&xS!|WC=NjiI) z%3W%2!6sBlZd5cF6O2i0tGcBzvXnUqH(7n1T63&msCB_ffbeZE{`+2u4!gw9N}$5|44#5)Ef5gn?^ zqXrh~SPNZXSq)H1dyA<8;+9+2N0`Y_Mc((bQy4aWnIhrj-P0gHpz3O1q7Kn9vpAhMA zaC~;m9g-gkCqIKfH+cEAJ+MuGfE3c$$Ws@WjNQK9WbVJ?F{_-`8sTeQ9 zb-D9o`!}r&S)6o^lx)a?J9O==tRMc#u!H@d$^aHqCwk#nZ87f4Z%g4^(OSX)r4G4p zr7=_Jbnn>AmEBzEy@!1Kqe=6i`=6AIZm9RWHG`_V*Ur_{(Oon;=cP#DpE!oI_uJ3n zQg_7Uy5X|-WMgpL?Jtw{Iy3Au;tCUggtd`4isz^(rNr}+z}}yW-P^4=&Dj|l!rY|D z!~2!JCzN+KXu~{gSkwS97L!#(v^oaiYYOU7^mR?4#x9(@riLlS#Ug79k0{#UCMGXG zfI7A@AM5lLtFwKL$QGyB6s57(xyBx!IK^21)L)!J>-um2qOWbydiThb7@ER&7-hth zSdViPq_Nk^s^rf^KGYHl^xE)XK%z??vV&h&ZBEQv12H6lal|$knobK4LFSYF?BUxYX!Dg ztKhn>?jE~5U2o+Uj;s*Ztm+7T3j2PilYQyce61uo68fiYVp67b01QgWAOcLD{W*Hg zAc$h_#*PC`7IZ8vKYY19Z#J2`4C{xb%5}i&_^Xp%gd?K zKYwFB-%71>k4NEMz=Y<)=PJ|@kAQQUxSQEZy?mfJo+SPKtommJ1FT%j?os-h!5`hs z^Ai&Sf3G{$F3@jbEIO_->4A)1!YZ?tND? z9SRw__vl%8bL9~&fGWX74uRZZXVjkm$t7cU7pD|h~1Etvygc-&@%GuU63^Pd8t{Gw%@y)7r9c^ z)g3+S43n+}xkBdLSUp_h)a+W^$EgA0Iw$A2DJAQ1>~4kU3OQ;(xQ)usCLdKLcL8NmGF z-=$G~sB-I}D_7B_^!~~P#Vbz+2r|6{MaNx_M{`#Q^4Ks-t7it+J%jHooFrIR`3tu2 zsRf+;FkNAE83w889F8EY=6hpartygNbsG&dE)4X?RlTHIvwCnTgds9VQgtq9`_U@W zdVL^>^U%SxEzICcXb1Y+trm&Uv+uhs9#)6{09Lwpu&u}T&PP!o!Pkb8>Paw7oPMX-$YA6{V zsXNUtlDja_9XwakDj=`*fm85cl@Do1E-X+GY{Gd8(c~m;vz8(9;{s%Vqpm`WRrqZxM zSijTQh_)&kKqpnM9Q_*gp!45#lG}%?@aP4DEMe%v{L`O`Rjvb$(;jxuzGz*(#}GXP zg!A1lpG*75cZlN;vN**%t8qf>G;!deCnMAK)c<0bf!aWW?mojT_YZ=vr3b&MrC|7o zovC)RA3Ws2!*}tN)R^;n<<;|XqUxvZEXX%E<`skO=f3g+=fVtPPHx`aCvOPStPtWE zF#5J{f?USt7pX_vm1}a4Ok!dQp`Vpjo1>frlxR8LFG6HFEMH()2At`o{yEK*|Cj_Y_c znGeGQ8ah+xa?Xs(p_!@FGQMl6Vcd5{M7L$sA5SKK7J?67$Ww1GQ?|%iSURn)#_&V% zG$=0i9IEP4L9z=)q14MP5~`2;cf#EK6*8hVUtV~3^EghPBeG{Y_D+Q*&A>rwq9+waSTc&JbJy=vwwNHl*wIXzh!T%D%mH{vVyE8mgqE)Kn# zmaPYY5%J+S@+i~NC-cuW%#-0<=}S;ruBWpsR`;o9r|?xH@*_#oKX)DPOTqdEX6PL(*@Lst))HLhUD7Xm?h0$GhHI zhk)XEj+I(JscnR#<`mhx$@Y?fMO~XX^T%2MFKV7&bD3e#a-t~Y2QvkqnitQ0v2DCk zZ11;@{IOL3I5-WKdD7ioYAgZKkq6dSNSQsIRZo{!tO~)Pj#W8`e9as6;-I6ugdXAg zTG5dlCP~N{t)~W@ioGW1fJT!fY3_cj~}=ce)x*1nY+E~SX%kK znM}+qY|MQKBJE7jUEC@bl9C%`i_9azHE5u7^4=37ds~44-B}9;fO@gB2G!(Dn(sb8 z9!HRJ40K)N+<-`GX#04cFVf-Dpb%@dB1)jKO=~?SC-25B*<9?uX2lR~0P48p@xO0J z$*QSqwUDPr;6uTfKK|#69G??8PF~AB@CT+;m&HOl!tT}jEV80#zVTwx$b_&5#sj2* zoN=n=*0sRmX9B4F+`5ao6i`|5@vlS6@=-9^__#4JPZt)nd0X(p9J2y>Fw=0zf_iS^ z_={gOE9!ndgGW(GyuF}YJ=F=QBEbx4eIZKP9HjPHnqYz7wx$P-OW>2DLS+$)faZ&* z%V>NfRc5-RtRY6`I#`x`>=ImZLIwWvOCTs_Xi@v&{aaITprGZStz6Xbby*Kk!5 z%WeN;YJV|9X-oEC@6L=T0~%#Y^6CugHlHWF4x7m$P}Pi$9jsC^GB>e-m1-1CXwb1R z5JP$%nyAY8sfY3O8}hH4LC3t4P4Lv)X6Ggx{;H*dbUT3X-#5jXZCZS7`__&A?Y5M! zd~|@rU_YU&OZUqsfN^GpWWRi6^5B5mcwEspgXF~N;=Fx8fmm*O0ViK9eM&G$v9U$5 z&x3*`S*DNFbdSYeGm0NzP3sl#nugSzLC z=HWwq(3rdEecoDs`Ycz>@2~xGMpmsa)gi>zGQnr*od~W+>bvw6D+aIolm1JdD4yp| z$W7J5x;0szd_{_cyj06MFCDwg4k~@s`9IV%kqiO(N$12-6U!;Sr?qeTSd3 zcGXC~O%-;Zd;)mm%B7ww#&H53wCx_(izdm#eWXWs>zkE9NC&y3>A4B#Al`%E^c;a&!cASC<20#@-(qw>|UfR^sG@7rRot?iKMCDWgZo2OP^f+~Bh@{alm z$SW;1;uO+G(#p>2G<0uhe+W(U%}43}^qlYSddPo!R`{%F6f4zDiR2o8+STyzU!Vr1 ziEpd#kI|pG{M(nS%ZGlXy_}mY(1O|hN;tJ0FQmx=T;P?3%A}eC)@TKN1rboQdcVK7 zj07GhDNN%Y@}*?1ssfzjeb>QLTySOf{+G(Pn}2G%*U!KGb+W!r$KuiV8x%7IMLwB! z{cTe*5#N)qU=dHe11)J05Hk1b-8rg>sztdQcTTLb69;t5=}*+xrD4%PD3UF_JDwg% zmo_dAT(@GV8YI&QRn^8#mqy>Dc-!E8nV~^bR}{PEU_As*tA3#<$&RK(Pk)?Dzv-m_ zD?gCCMBh2pd1lY_a^L10Ys=-GekmCpcdLXo1D;zfB;b4ar|xTma+lRman$i~*{B=y z?Ymy+IX?BzSRbAa)%u&?U^6{F{c(;k;>;>8zH}Z7DdgX*;_iVyg zHAz!ymnSZIq9-Md!6D;>r6njIw z%I1pdbP{n|?-F~oe0{5MfB(lk-Ad&U+N{#nUpH?ifwhuv9{Wssttj)HdsQ>vWkYar zc27)inv2E1IX|2`k>PQsq@JD)nip4zGT}?RX`&)~;5eMy?vOHCLjJ90Dz1?Js}7`A z4#RaH$J}9QsUYK|4hM>rK1&;V^s`D%HpK#_^;zn7(y=W2={N9T*QZ5sQq$HkwwHhb zTSRJR(?AWwT`I(78C(JhV zU+kB)qO+V=lay9v@L7$DtEsE)D)v+njX6PXNoRyXWw}Tg$^N%^vftWS!1VB1{v+<# z#Xb^mep#!z|7MWx6qOVDb>WvzGkn;sm8WWKHdYU_cLv?|l9*DN|tMTMpD>A}}4 zFYic#L2@m;6~`&u_QaaNSadcRxlQ<5(R@Y`m%Lulww@AK<-WpALHYCd04*7Ohv-Zt zU6tz|FV_s_FOR-^^`b@U$L1Vuk*j|97;ib4lZghhe|_tx$X>MwuDhU>8h8-;Ktc9I z-iMB45AUkIq|K?OJKz3-q}?qHLX=z~#>R{VA-*�p7q|K)6Z;o<14A{h8&*4e#}W z&DnZA!PDv>>Y534ZY0PIG+P;e;Oh*#)sW@%{o%$ue?#Oc7_ej0ChAzT&^!=s&2sfW z+wfV{=y;ra5wm7sYlrWeC z1sV|3KrOCICr~Bxc=aGb*6xQEAjJ{WLgpOV%w*}oXuV$2-6$2Av5epG5 zOBisuNwcY3VKuAlKNz&-4+PW_6N2vrnh^(FJ-`V}Q{0ig`r(hg$(<^{L^&dhSOG>h zYF@;%szGXF!zM6}PdrjKnJuR8eb(5LK7Lz&w1fQncJqVdxZvfqbQKxf0n(Q739qYE zkiZsbVk6~EFJ)f5FApYGC~zc4T_$vLVQwO%rfm`)8FlI*W3SUdSsL2bt-vIRU5oB!w*@Nr%oNZgue)SuR&pf|xp z{Y!!tW=B!?86q^QoTu+PKP{27($S&hMy5fdejrO}c%&7w7g8zJzR7RJ#zebRDkY=+ zOg9@8s6K$>evMD#5;hs$P5*ACXS*Eu4(L9Ieq%vl6)(`@{InADdW1~niGzsn#yIgpV98$jX)OCO)! z3qA8bNt*gpJ||u4!N1JXX{!4mi@td);KZ?wy#5s{Is>&i0{PZu;j`X&(&v?Lf3un~ zQ<){SFYkk+yqmY`8A!zUQQZxz&4j_>q(L^S`)rvR_b8z zTHXcD-T+R?xUWBR^K@^?uLYCA=@l?^t;)o)yOTB8v(Ah)Nz%E)CqRGbkrJPo;(=qJ zZ4VDN1qs>%ei^JPhb0*0OoBpqVYh9@faw}Gq&nMr??j`sSd z9l8_j6fSqc?q6ivn^AjP*VdJlW0#9FU1>rsnR4un(JF$3IR_ zk`B1@dJjR$7=fUF>qg1)1BG%J? z)k??s=&pcp0?!d+#P18XE8Fz|M*jWK0-UtgnHH`c8`1t){aLTbrJ#_!-fo6JgowDc zpNQ#z7BSKLdqVtW|W@~UD zxEdHe6Wl^mMsNQ6{C&Aw1ZOT49;1Tp0GKRV13v#E%rx<--k^1fCmy(qSP;g+AtE4MLmEA-A6k7ZG*n*!YNTcRz6yfeH}*KC*_mkVdk;*8 zeJC!-e(<%F);5)Qs?w?M|$d{>SD7e1G_B~$g1@7N|} zaX_4@CT|G`3hvYwt$YiQOi}fY36weTq_Y2x3|THB9D6bWCOVn-uAv zG1oQi_Ie)(Q!_t;5emz0hyZ|UU&_2H4XSv92}HUcnZ&Y~R&l1ReN4ny{j|El*A}>a zB>;+(b^BP42w?WM(XEH%!rl5F)fri+bx=@O0v)75j&^%D``IqA2yasomyTaJ2W9Up8*=~OaM zE2y?Cqf^~4JU*nK5{c6;R5&WzoNUt*jQPbM0kqrg$?vn{T>&dHSFPS`5hd9Z-G4W% z$wJdNi~|3xpSzUaeB&+{w8S|4&%%P*+aJ`uuytit|7fbTjrz5LFBVz5=i~;65Tbaw z;UfJmfJ*^5SHy>}zq2*p>US~;hAKCNcF%qN_94@1qHvITU0G2VVmb}8_{>|9YJD|gO*Rc^A0Pa;3 zZqS{I$(-@aj9=yi;%mtNemYQuEqj+`@Xk#d9#wBs|E-xPEh^w%M1@#prx08*H1>GE zba>RYe9h;>N#Qt>Ivilghb zi)|$z()GS{kW`gT`cHe)G)v6;;gShGDp`~4@)Ut3!wx9u2Z@4&(tI3|llW@6e zcvh!xXZsAH&U}Lk;|eOvkVJXo>=f67q&Qbp{D*}u8-ZF(dmDNF{*C9)SwOcZRSVsG zarJieS(7N>b~;uk$pKW43zxvS+}Q3{n-f^jxaGwV!Z~(EPhzzWNDa>Dw<`01iVqj`ro7xmOW{5aC?yv7AXDECJhr6}< z04Nz%!{5LKfHy&WUs_q}l0lXwaMQ7XZ>N-s=*Y@_q_7%raB>RxZGofFs#=3YPiPT)c0(snSqg(fe?-Vap%@Xe+W7*)JZZ8i!{&Y+$3l6*#(psr@ zh_3@IBIs|Jfto763!}~}ePIEBrV-zl@h|p%I`8L#)a1wFJnK$#%8@Gf;{z6Z5VE|& zV0V7*MdrHN+h>yUbk{gQM0c^!nu>kF14{PMMP#=S|faDYafJ#RnR zBY54OIAXD-@f^|N;eqZsNP|9ybW(B? zP&65fDwQv zJ?%(FY*7|Xy5*S|$gcWPr7jS`X3-0m9C?GC%ucIwWO9x!4z5Rerty}b)1O`HEUZ^0FH`8i7l z7#d_d$Fo@<#A$T}P=R={2~r=0Khckc?8nj^@CIr+p_zctA^{t`|1;Mtl3L(Ro&*yB z#w+B&#=Pvx+l{6+JC$hj;1Xz85DTZX ziXWOQo~EA5$aE?_xmsi*bmhoOA$BMT$^m_|UKJ$jh@QbFt^og9JeeOc0|hDhrc1vWUb>e|u!@r+~67A|V%Xlar3i~k%Z2t)?vJFi&Iwv&Jq zG?thC^(+-KE-(De9>S|I?(f?hhe*3J7umeJm7@{YyQX^zPBSG9UF}y^(Vs>hWKP)=!!&L%(+DovV zk<$A9`cKd=_7)-ATcH_gTaJir9Qj-|y!|0%Co_$kiw5RnqIPH#pjd)up^oKNa=QQ# zw#O?+Q0>^{_X1ty^J+guTI{>`^i5YF`ysV*6P14h$k?m1CZ0#bk>o=GSQ@}+3WI** zbK-W7{t(ND(dip;%d|&6_?zvEUqO2By~;9w4Zwgfjz8X+81BmxUBMe+`5} z+Y1UA5RxKYFXG2XOH&&cL3)jd1ahPY9)q5_b%HXcwXCKtv z+Q9`4gr1epT7s+tUXZe!7zKw}mz#K?9P%u^M-?cO|M!gag)RAplNgQ+%V7p{UqF#z zTJn^^|JNR4DXX5Nytz-Nj`pBnLrh)Ua~}kQfLLI0}rHI zj$J6@*}!P(G4a(J`-dXfoibaExQiAcyIZ8*D1RJ z;$s4BQ=kR*zfL=_I^~I&*p>*;sKn-GDj=&abwE>ejLC1~9c{VUG_zuQrMAODzo$!?NFPr(S$ zk5Pm<_hEGO=kL;>goKdHqgK1EypkeK)5Q^jKNTOy*efE6gl|FT&Wysc%vx+k*rG3F zm+?r{-%GIr1W%UUp+}s@e#u$Ay$SJ{*Z|1M8qKW4 zS1xIFB(_V4)B{{$yq$gex1b@$3xD&U|xvysO8Z2{5p--n}r2gGasC2i66Mu$Q9H+9v))X*=Iel7ycFT zN{BP)lUzg4Gt_|X``Se(E&J~yk5xmDn2&UF8lmu>PG2GomwQmI@%AeY6UKg z4L{nkJ@P8ooYk9df>_JPB_z-U#qRFU1peqP3^7Q#y4fn%p);sdE&PBD3c+|;u?*;xKGJe|?SbJ1wki zY4^GPZdH`7vok2dKw6bgB?Uic%_fddRMNN2y0o5&%14tz7?g^r+SkEX{G-E2W=@)h zSa7fanstHQYQbhSZR*F|-CbiY%1u~=huW1gP(2*Z^)okEvbkRpvikQERN?X|CI7hF z_aTD&WAnnzWQD*oz+qnv!I%o;wT<4x(NWQyvdX*tht$TsR`~`n+cyl*fW?2;LoZ<$ z%&qOF78%@=_j^JJ6YMl)zN6*NcM;@Fn(LIjQ>hfU3 z87zxo$wo)-YFTQ{U|I8FXZgwp0@{L(vcNS$lg6iq%d&y%TL7v#0sqa5fy$#gM)d+P z%EwvTHJm5S9t$uYmjii(XteF5k>b8#m!FMrmQX_mTJ@w+DFiI(%)3{mVXb)w@rI*m z@oHJ4zSi+bMRrujPO} z>FnWAaj^V-FOdO?G=vI9Y??VCaAW;1xaMJFBt!mX>WqkFKa1@7_R}>dD+`=dAd1({ z9?8k&=b#R9{G8izYbSlR{jpy6VsR#AArmeJ^EdWAk2iXq3iDNqA_i_dAW}axV8k2U z@t~gwvbN%=Bl&a(+1>Uy!au-LF12EBn7;SYleJ6t=#n{9Tn$A-&O5p&bit4H$z zhy0dudU`e)4`puDv#;iT)sTrXdxk88Cp3d)-Sh@_{E#hgI&NatM%O~_2U)D z;d>-8rh@sG3_tt~J=n1gOdc1KDMs4K8gb2E|9?%^h%L&^1fNh1O{B8&YQ6ldw_d)o z;?%hJ5jkY~f(j2bud2rnpg{(3*MD{Ov0onynwN~X(zpFG|AD+W^t=-mdNyw9eNQb*rl0Q58`Ow@ zHa05YPH%c*6~u{JUFdbdyQL=&5Xw1hEl|T5vjKBMQ}+GaSeK9%IsNk4h-%?G9BHP6 zLH=J@+zK&%U4eg|7VG2FBSCHX6d=##b=Mn2mS3N7(;G0Gflch+8$o&HGP|8zJLw+T zDMWq6864S7T0w8OTqZLwBSYWW457QW1t-!ofrAvU`$4wj)Xk&L<$Dc&_sUxP^cQ&p zwBI}RysNc&ar9#D*;i=L`>Yy2>op071emnp15@E%<4~Pu>t91JLsmIZ{Ze`^S>*{G zK(U5et#kR`-Mxo=g9$8qvZYZigV9SzJzTSCb7{tpAu+vtNZN*xE{U=@5cyN2sm<=o zkN*yD=G(+(!BMpy!aQ{ctKYFy>ut;3qGKvGSoOJywFDg!c6rWFKXc=u86vVjt;zDu zDelK8CVMXBdHQg7gSiJnpvlz0C`A2><*xxp@@GeK`ufuJ8c^o;_f+Kr^%^Zaqgz!F zatKK@nZ*TmcmHLQ%E_D!fbWf5aUC4r^1~{>P`kkzx8#@L%D+8A7C_aQ6oUU0B4nLD zbf^Ud;m-T-I>_1`fF9)bT=l>Wgr(8Mg&eim% zGzYBs6ss{%gw$xlS0fYym&(p&#)D8j{JK31bEP5Z?%W@518a>(Ej6nhioR1CE8ssB z@CC{6OalM94>Z_G8~kEcQF-CF`^(0^?@B;gBi%@d1BbmD0H3(FkK@qAMfy6ruGw-M z`oiaSeoF=YToE0aCoh(fZ0qp}94ZY}gNwT)5Elm1GwJ^S`%Ka@i5c_{E0!tduR=lnqbNNp~Df}JD~Gxh7a6A8fi=qP1m79r*RZ~>6VW!G&V zu5SmbzJt8j`}}Fy!E5z(au!!ZZ*E;+o_!D%45|kY0zw-!MHAV}Y~(tHGWZLS--Gso zq5Pg+iRk6!%>TK7%eQS%_1kmu;}($zzGwJaWHRpuYgIE~O-t#>9`)V! zQ9<}%Nc0K8<{+?q*!od=t(5*;=Sq&!xxrj1xKmv0RoykVw{CvKLouJNiIaJ~YALiD zsA22Ukujt0!lK17b(Quk&C&TbRDPSV$bVt3k++j>F6I3ky+T;$VnQRMNjD;F^Q|9u z3rkzkznoF?`p=)=^d42jNxAI&p8yI6_4ps3ieHqH0HAfW3fBDrCa3Cm0(Pbc{%Nz8 zNI?N-WI2_-f3Lgj1%j>CeSE!WPv6H6hJqp-oCEuLUj6TKNyXZSn^N)1QWgL-|N2!I zt6%vm_40;P{$9Y2t{HQ)Tz}&f@pw}DuMyHn-@k8fywmnxr#(IB(`gS5uv@&z`4y`l zK9;IqnDS396#_=8U#k};a616*r3U>Ov+t5&7hPt$5}bz~)K53wskKhZJ1K2ywjU0M z)Zfu~2ad5%B?Migc&IOH%#&0Xrxu?JcwC-7Uo>wEwjm zz29D6)~~CtmddIV(%ROfz7DZcA5L{rJr=0Je|xHbaayij>JZ@dddH|)(zk^x0dS#F zS6nAI-#W`W1{h}P8Gd`yom#5_PfKgF)oCvlkC`qF=qWw(UzLu*UrXD~ODhEUyqBxD zctThc0GGM-w)t}TsB2A^k(G1{-KOR`>)yS-kKf#?X8551Tl3&H`e2_`slM!grnIkK z>R1@~Gxd_q(r|^GoLpJ3_&({?tDosovVvZzuTRHoXsVOSs^e1IP$NkW&Vjpa6mRmL zR4({Cn~W@9?${#uXW@1LT%fG1j|u>*B{w(EbSbXYs%G{2Slp`W6V_gQ`sgEDBt>5K zNb-(UZS^~!jO}0QXcqKUp<^H%ZuB+d<;Dp!Oqby*dcZ$fd0grnYo$>=q73-RPATc| z*I60g!0caNzEVBuATT7;?+x^nUi3F-!AiRSb(6lFueapsPJ0baweRi4*W2=dEOt~q z{;j9I{kzIY2NVDLRSVTyKOig)gu@gSm&(G$tLgsNNr_}aBH^&rX;0t7Z)$2E@b%XK zEc!^_e^P`0#!SDlu@+`31YE7&{u}D$6!pk8RR_=&Lxx@=Gv_X+``p0xz zmPYH6zHYJG)Z8ev)&<^2-{;at`F=_b{zps{VDwY0@WGXZ>Vs5Y4fvM&a-hjtfX)~< z{x%si_7>A+*oxkM7f=I!ePfMOR#iw_TZ`3*ht|5lIaR91zlb-vpP74)F`a=7VwfDh_FV2I3~zryMRkWKVaxXmpM($LVp zXs@NU$y&%4r_&zL6^GR0D{9vNn40wmOc$fsCIFn)(KFy*)yuzE4^~>iHm)9XlZ?M< zn&~pqE*6WaLB3Jy)qr1BbHeJW51`xhCH_a$jQ^9S3;7`11%T81>sO6e<$XoHoNBrj zK$|w}PPusaDAQ%6oIWI{wXMavtgi=reQKU=w+o<0^uI*Ui6{AB#hQm-H(kux*fs!Y z1rMz7d41lSRGI74m-9^5gY2bO{oj4h=OjPBz;r1+rWfbwOZum(Dx|KiN}5_4xttHA zl1Kii{!RNgHSoV_x|H*=bpSZ6BM3}a<$hFsIafV6gB2Mybl9abbM7+JWptT75icAH zN^77+8XN1{Ps~$;eEkQ?S#E^RN0Hwm$TL5 zBGWY?JKQp9j*Pl;tm!f?NUy}#Tk!O@Jl#^a{lvU_YtbHnw9tF_-%6^1|HPVyo;O`g zS2!vFoXP+1}X&~y(0kIz@)k>oh_|BCJ*FjGDHtB22UX>gIUUVUZp zQhoQy9{;-NPJ4ksn^f1Hk{b2Uef7gC1kiW~NGts>nBEferury&C@IO^C)Pad=%j6r zPqY+h4+GvI9`TG;U(Q#LiRy8a;j-WYCcFQs4;cvs1MRc>j)QY_o84eYciN-z4v=Pg z3!k3dKc_~^r&R*jb7IXy6{d?y71soSGqo=o^z~PtkXMSwGeP~)t?F^D&|nDNF#nEw zttLR3qG$7sjrG#lTxTuZ)0gx0eSNm(0U2>p+7Ia2sU9z>QSwN|num@Wt|={C834{? zT^aCti&YR9>`5kvtLHK5kM(M<%hls5VMhVZLwEgKaqs8E=gUdkdqiUqX=!Pa`i2^7 zOP)SFM-TY1n6Vb&K(?rr_8I;GH5%@cr0i4A`$du!6>IF#tuLKi9{?_-;KB0YUXP4Z z|0IU1|MEvl`&EM8>Y-PB>4k+Q>XF~^I=gznLIw{RZnghSCB5ffx6`dt1Aa|yrPS3| z(MKOh9i5y*9k2BO-_r5*DfKv-6glkiCXY%&UawgDaFwYd?co0bJqDQ?R(SJw00000 LNkvXXu0mjfM&xy; literal 0 HcmV?d00001 diff --git a/crates/lsm-tree/renovate.json b/crates/lsm-tree/renovate.json new file mode 100644 index 000000000..39a2b6e9a --- /dev/null +++ b/crates/lsm-tree/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} diff --git a/crates/lsm-tree/src/abstract_tree.rs b/crates/lsm-tree/src/abstract_tree.rs new file mode 100644 index 000000000..74f1a1f3b --- /dev/null +++ b/crates/lsm-tree/src/abstract_tree.rs @@ -0,0 +1,609 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + iter_guard::IterGuardImpl, table::Table, version::Version, vlog::BlobFile, AnyTree, BlobTree, + Config, Guard, InternalValue, KvPair, Memtable, SeqNo, TableId, Tree, UserKey, UserValue, +}; +use std::{ + ops::RangeBounds, + sync::{Arc, MutexGuard, RwLockWriteGuard}, +}; + +pub type RangeItem = crate::Result; + +type FlushToTablesResult = (Vec, Option>); + +/// Generic Tree API +#[enum_dispatch::enum_dispatch] +pub trait AbstractTree { + /// Debug method for tracing the MVCC history of a key. + #[doc(hidden)] + fn print_trace(&self, key: &[u8]) -> crate::Result<()>; + + /// Returns the number of cached table file descriptors. + fn table_file_cache_size(&self) -> usize; + + #[doc(hidden)] + fn next_table_id(&self) -> TableId; + + #[doc(hidden)] + fn id(&self) -> crate::TreeId; + + /// Like [`AbstractTree::get`], but returns the actual internal entry, not just the user value. + /// + /// Used in tests. + #[doc(hidden)] + fn get_internal_entry(&self, key: &[u8], seqno: SeqNo) -> crate::Result>; + + #[doc(hidden)] + fn current_version(&self) -> Version; + + #[doc(hidden)] + fn get_version_history_lock(&self) -> RwLockWriteGuard<'_, crate::version::SuperVersions>; + + /// Seals the active memtable and flushes to table(s). + /// + /// If there are already other sealed memtables lined up, those will be flushed as well. + /// + /// Only used in tests. + #[doc(hidden)] + fn flush_active_memtable(&self, eviction_seqno: SeqNo) -> crate::Result<()> { + let lock = self.get_flush_lock(); + self.rotate_memtable(); + self.flush(&lock, eviction_seqno)?; + Ok(()) + } + + /// Synchronously flushes pending sealed memtables to tables. + /// + /// Returns the sum of flushed memtable sizes that were flushed. + /// + /// The function may not return a result, if nothing was flushed. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn flush( + &self, + _lock: &MutexGuard<'_, ()>, + seqno_threshold: SeqNo, + ) -> crate::Result> { + use crate::{compaction::stream::CompactionStream, merge::Merger}; + + let version_history = self.get_version_history_lock(); + let latest = version_history.latest_version(); + + if latest.sealed_memtables.len() == 0 { + return Ok(None); + } + + let sealed_ids = latest + .sealed_memtables + .iter() + .map(|mt| mt.id) + .collect::>(); + + log::debug!("Flushing sealed memtables {sealed_ids:?} to table(s)"); + + let flushed_size = latest.sealed_memtables.iter().map(|mt| mt.size()).sum(); + + let merger = Merger::new( + latest + .sealed_memtables + .iter() + .map(|mt| mt.iter().map(Ok)) + .collect::>(), + ); + let stream = CompactionStream::new(merger, seqno_threshold); + + drop(version_history); + + if let Some((tables, blob_files)) = self.flush_to_tables(stream)? { + self.register_tables( + &tables, + blob_files.as_deref(), + None, + &sealed_ids, + seqno_threshold, + )?; + } + + Ok(Some(flushed_size)) + } + + /// Returns an iterator that scans through the entire tree. + /// + /// Avoid using this function, or limit it as otherwise it may scan a lot of items. + fn iter( + &self, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static> { + self.range::<&[u8], _>(.., seqno, index) + } + + /// Returns an iterator over a prefixed set of items. + /// + /// Avoid using an empty prefix as it may scan a lot of items (unless limited). + fn prefix>( + &self, + prefix: K, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static>; + + /// Returns an iterator over a range of items. + /// + /// Avoid using full or unbounded ranges as they may scan a lot of items (unless limited). + fn range, R: RangeBounds>( + &self, + range: R, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static>; + + /// Returns the approximate number of tombstones in the tree. + fn tombstone_count(&self) -> u64; + + /// Returns the approximate number of weak tombstones (single deletes) in the tree. + fn weak_tombstone_count(&self) -> u64; + + /// Returns the approximate number of values reclaimable once weak tombstones can be GC'd. + fn weak_tombstone_reclaimable_count(&self) -> u64; + + /// Drops tables that are fully contained in a given range. + /// + /// Accepts any `RangeBounds`, including unbounded or exclusive endpoints. + /// If the normalized lower bound is greater than the upper bound, the + /// method returns without performing any work. + /// + /// # Errors + /// + /// Will return `Err` only if an IO error occurs. + fn drop_range, R: RangeBounds>(&self, range: R) -> crate::Result<()>; + + /// Drops all tables and clears all memtables atomically. + /// + /// # Errors + /// + /// Will return `Err` only if an IO error occurs. + fn clear(&self) -> crate::Result<()>; + + /// Performs major compaction, blocking the caller until it's done. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn major_compact(&self, target_size: u64, seqno_threshold: SeqNo) -> crate::Result<()>; + + /// Returns the disk space used by stale blobs. + fn stale_blob_bytes(&self) -> u64 { + 0 + } + + /// Gets the disk space usage of all filters in the tree. + /// + /// May not correspond to the actual memory size because filter blocks may be paged out. + fn filter_size(&self) -> u64; + + /// Gets the memory usage of all pinned filters in the tree. + fn pinned_filter_size(&self) -> usize; + + /// Gets the memory usage of all pinned index blocks in the tree. + fn pinned_block_index_size(&self) -> usize; + + /// Gets the length of the version free list. + fn version_free_list_len(&self) -> usize; + + /// Returns the metrics structure. + #[cfg(feature = "metrics")] + fn metrics(&self) -> &Arc; + + /// Acquires the flush lock which is required to call [`Tree::flush`]. + fn get_flush_lock(&self) -> MutexGuard<'_, ()>; + + /// Synchronously flushes a memtable to a table. + /// + /// This method will not make the table immediately available, + /// use [`AbstractTree::register_tables`] for that. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[warn(clippy::type_complexity)] + fn flush_to_tables( + &self, + stream: impl Iterator>, + ) -> crate::Result>; + + /// Atomically registers flushed tables into the tree, removing their associated sealed memtables. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn register_tables( + &self, + tables: &[Table], + blob_files: Option<&[BlobFile]>, + frag_map: Option, + sealed_memtables_to_delete: &[crate::tree::inner::MemtableId], + gc_watermark: SeqNo, + ) -> crate::Result<()>; + + /// Clears the active memtable atomically. + fn clear_active_memtable(&self); + + /// Returns the number of sealed memtables. + fn sealed_memtable_count(&self) -> usize; + + /// Performs compaction on the tree's levels, blocking the caller until it's done. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn compact( + &self, + strategy: Arc, + seqno_threshold: SeqNo, + ) -> crate::Result<()>; + + /// Returns the next table's ID. + fn get_next_table_id(&self) -> TableId; + + /// Returns the tree config. + fn tree_config(&self) -> &Config; + + /// Returns the highest sequence number. + fn get_highest_seqno(&self) -> Option { + let memtable_seqno = self.get_highest_memtable_seqno(); + let table_seqno = self.get_highest_persisted_seqno(); + memtable_seqno.max(table_seqno) + } + + /// Returns the active memtable. + fn active_memtable(&self) -> Arc; + + /// Returns the tree type. + fn tree_type(&self) -> crate::TreeType { + // NOTE: This is only really safe to do, because we validate + // that the config's kv_separation_opts is consistent with the tree type during recovery. + if self.tree_config().kv_separation_opts.is_some() { + crate::TreeType::Blob + } else { + crate::TreeType::Standard + } + } + + /// Seals the active memtable. + fn rotate_memtable(&self) -> Option>; + + /// Returns the number of tables currently in the tree. + fn table_count(&self) -> usize; + + /// Returns the number of tables in `levels[idx]`. + /// + /// Returns `None` if the level does not exist (if idx >= 7). + fn level_table_count(&self, idx: usize) -> Option; + + /// Returns the number of disjoint runs in L0. + /// + /// Can be used to determine whether to write stall. + fn l0_run_count(&self) -> usize; + + /// Returns the number of blob files currently in the tree. + fn blob_file_count(&self) -> usize; + + /// Approximates the number of items in the tree. + fn approximate_len(&self) -> usize; + + /// Returns the disk space usage. + fn disk_space(&self) -> u64; + + /// Returns the highest sequence number of the active memtable. + fn get_highest_memtable_seqno(&self) -> Option; + + /// Returns the highest sequence number that is flushed to disk. + fn get_highest_persisted_seqno(&self) -> Option; + + /// Scans the entire tree, returning the number of items. + /// + /// ###### Caution + /// + /// This operation scans the entire tree: O(n) complexity! + /// + /// Never, under any circumstances, use .`len()` == 0 to check + /// if the tree is empty, use [`Tree::is_empty`] instead. + /// + /// # Examples + /// + /// ``` + /// # use lsm_tree::Error as TreeError; + /// use lsm_tree::{AbstractTree, Config, Tree}; + /// + /// let folder = tempfile::tempdir()?; + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// + /// assert_eq!(tree.len(0, None)?, 0); + /// tree.insert("1", "abc", 0); + /// tree.insert("3", "abc", 1); + /// tree.insert("5", "abc", 2); + /// assert_eq!(tree.len(3, None)?, 3); + /// # + /// # Ok::<(), TreeError>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn len(&self, seqno: SeqNo, index: Option<(Arc, SeqNo)>) -> crate::Result { + let mut count = 0; + + for item in self.iter(seqno, index) { + let _ = item.key()?; + count += 1; + } + + Ok(count) + } + + /// Returns `true` if the tree is empty. + /// + /// This operation has O(log N) complexity. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// use lsm_tree::{AbstractTree, Config, Tree}; + /// + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// assert!(tree.is_empty(0, None)?); + /// + /// tree.insert("a", "abc", 0); + /// assert!(!tree.is_empty(1, None)?); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn is_empty(&self, seqno: SeqNo, index: Option<(Arc, SeqNo)>) -> crate::Result { + Ok(self + .first_key_value(seqno, index) + .map(crate::Guard::key) + .transpose()? + .is_none()) + } + + /// Returns the first key-value pair in the tree. + /// The key in this pair is the minimum key in the tree. + /// + /// # Examples + /// + /// ``` + /// # use lsm_tree::Error as TreeError; + /// # use lsm_tree::{AbstractTree, Config, Tree, Guard}; + /// # + /// # let folder = tempfile::tempdir()?; + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// + /// tree.insert("1", "abc", 0); + /// tree.insert("3", "abc", 1); + /// tree.insert("5", "abc", 2); + /// + /// let key = tree.first_key_value(3, None).expect("item should exist").key()?; + /// assert_eq!(&*key, "1".as_bytes()); + /// # + /// # Ok::<(), TreeError>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn first_key_value( + &self, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Option { + self.iter(seqno, index).next() + } + + /// Returns the last key-value pair in the tree. + /// The key in this pair is the maximum key in the tree. + /// + /// # Examples + /// + /// ``` + /// # use lsm_tree::Error as TreeError; + /// # use lsm_tree::{AbstractTree, Config, Tree, Guard}; + /// # + /// # let folder = tempfile::tempdir()?; + /// # let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// # + /// tree.insert("1", "abc", 0); + /// tree.insert("3", "abc", 1); + /// tree.insert("5", "abc", 2); + /// + /// let key = tree.last_key_value(3, None).expect("item should exist").key()?; + /// assert_eq!(&*key, "5".as_bytes()); + /// # + /// # Ok::<(), TreeError>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn last_key_value( + &self, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Option { + self.iter(seqno, index).next_back() + } + + /// Returns the size of a value if it exists. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// use lsm_tree::{AbstractTree, Config, Tree}; + /// + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// tree.insert("a", "my_value", 0); + /// + /// let size = tree.size_of("a", 1)?.unwrap_or_default(); + /// assert_eq!("my_value".len() as u32, size); + /// + /// let size = tree.size_of("b", 1)?.unwrap_or_default(); + /// assert_eq!(0, size); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn size_of>(&self, key: K, seqno: SeqNo) -> crate::Result>; + + /// Retrieves an item from the tree. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// use lsm_tree::{AbstractTree, Config, Tree}; + /// + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// tree.insert("a", "my_value", 0); + /// + /// let item = tree.get("a", 1)?; + /// assert_eq!(Some("my_value".as_bytes().into()), item); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn get>(&self, key: K, seqno: SeqNo) -> crate::Result>; + + /// Returns `true` if the tree contains the specified key. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// # use lsm_tree::{AbstractTree, Config, Tree}; + /// # + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// assert!(!tree.contains_key("a", 0)?); + /// + /// tree.insert("a", "abc", 0); + /// assert!(tree.contains_key("a", 1)?); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn contains_key>(&self, key: K, seqno: SeqNo) -> crate::Result { + self.get(key, seqno).map(|x| x.is_some()) + } + + /// Inserts a key-value pair into the tree. + /// + /// If the key already exists, the item will be overwritten. + /// + /// Returns the added item's size and new size of the memtable. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// use lsm_tree::{AbstractTree, Config, Tree}; + /// + /// let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// tree.insert("a", "abc", 0); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn insert, V: Into>( + &self, + key: K, + value: V, + seqno: SeqNo, + ) -> (u64, u64); + + /// Removes an item from the tree. + /// + /// Returns the added item's size and new size of the memtable. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// # use lsm_tree::{AbstractTree, Config, Tree}; + /// # + /// # let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// tree.insert("a", "abc", 0); + /// + /// let item = tree.get("a", 1)?.expect("should have item"); + /// assert_eq!("abc".as_bytes(), &*item); + /// + /// tree.remove("a", 1); + /// + /// let item = tree.get("a", 2)?; + /// assert_eq!(None, item); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn remove>(&self, key: K, seqno: SeqNo) -> (u64, u64); + + /// Removes an item from the tree. + /// + /// The tombstone marker of this delete operation will vanish when it + /// collides with its corresponding insertion. + /// This may cause older versions of the value to be resurrected, so it should + /// only be used and preferred in scenarios where a key is only ever written once. + /// + /// Returns the added item's size and new size of the memtable. + /// + /// # Examples + /// + /// ``` + /// # let folder = tempfile::tempdir()?; + /// # use lsm_tree::{AbstractTree, Config, Tree}; + /// # + /// # let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// tree.insert("a", "abc", 0); + /// + /// let item = tree.get("a", 1)?.expect("should have item"); + /// assert_eq!("abc".as_bytes(), &*item); + /// + /// tree.remove_weak("a", 1); + /// + /// let item = tree.get("a", 2)?; + /// assert_eq!(None, item); + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + fn remove_weak>(&self, key: K, seqno: SeqNo) -> (u64, u64); +} diff --git a/crates/lsm-tree/src/any_tree.rs b/crates/lsm-tree/src/any_tree.rs new file mode 100644 index 000000000..29cfbaf6b --- /dev/null +++ b/crates/lsm-tree/src/any_tree.rs @@ -0,0 +1,17 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{BlobTree, Tree}; +use enum_dispatch::enum_dispatch; + +/// May be a standard [`Tree`] or a [`BlobTree`] +#[derive(Clone)] +#[enum_dispatch(AbstractTree)] +pub enum AnyTree { + /// Standard LSM-tree, see [`Tree`] + Standard(Tree), + + /// Key-value separated LSM-tree, see [`BlobTree`] + Blob(BlobTree), +} diff --git a/crates/lsm-tree/src/blob_tree/gc.rs b/crates/lsm-tree/src/blob_tree/gc.rs new file mode 100644 index 000000000..a905faeeb --- /dev/null +++ b/crates/lsm-tree/src/blob_tree/gc.rs @@ -0,0 +1,307 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + blob_tree::handle::BlobIndirection, coding::Decode, compaction::stream::DroppedKvCallback, + version::BlobFileList, vlog::BlobFileId, +}; + +/// Tracks fragmentation information in a blob file +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct FragmentationEntry { + /// Number of unreferenced (garbage) blobs + pub(crate) len: usize, + + /// Unreferenced (garbage) blob bytes that could be freed (compressed) + pub(crate) bytes: u64, + + /// Unreferenced (garbage) blob bytes that could be freed from disk + pub(crate) on_disk_bytes: u64, +} + +impl FragmentationEntry { + #[must_use] + pub fn new(len: usize, bytes: u64, on_disk_bytes: u64) -> Self { + Self { + len, + bytes, + on_disk_bytes, + } + } +} + +/// Tracks fragmentation information in a value log (list of blob files) +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FragmentationMap(crate::HashMap); + +impl std::ops::Deref for FragmentationMap { + type Target = crate::HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for FragmentationMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl FragmentationMap { + /// Returns the number of bytes that could be freed from disk. + #[must_use] + pub fn stale_bytes(&self) -> u64 { + self.0.values().map(|x| x.on_disk_bytes).sum() + } + + /// Removes blob file entries that are not part of the value log (anymore) + /// to reduce linear memory growth. + pub fn prune(&mut self, value_log: &BlobFileList) { + self.0.retain(|&k, _| value_log.contains_key(k)); + } + + /// Merges a fragmentation map into another. + /// + /// This is used after a compaction stream is summed up (using the expiration callback), to apply + /// the diff to the tree's fragmentation stats. + pub fn merge_into(self, other: &mut Self) { + for (blob_file_id, diff) in self.0 { + other + .0 + .entry(blob_file_id) + .and_modify(|counter| { + counter.bytes += diff.bytes; + counter.len += diff.len; + counter.on_disk_bytes += diff.on_disk_bytes; + }) + .or_insert(diff); + } + } +} + +impl crate::coding::Encode for FragmentationMap { + fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error> { + use byteorder::{WriteBytesExt, LE}; + + #[expect( + clippy::cast_possible_truncation, + reason = "there are always less than 4 billion blob files" + )] + writer.write_u32::(self.len() as u32)?; + + for (blob_file_id, item) in self.iter() { + writer.write_u64::(*blob_file_id)?; + + #[expect( + clippy::cast_possible_truncation, + reason = "there are always less than 4 billion blobs in a blob file" + )] + writer.write_u32::(item.len as u32)?; + + writer.write_u64::(item.bytes)?; + + writer.write_u64::(item.on_disk_bytes)?; + } + + Ok(()) + } +} + +impl crate::coding::Decode for FragmentationMap { + fn decode_from(reader: &mut R) -> Result + where + Self: Sized, + { + use byteorder::{ReadBytesExt, LE}; + + let len = reader.read_u32::()?; + let mut map = + crate::HashMap::with_capacity_and_hasher(len as usize, rustc_hash::FxBuildHasher); + + for _ in 0..len { + let id = reader.read_u64::()?; + let len = reader.read_u32::()? as usize; + let bytes = reader.read_u64::()?; + let on_disk_bytes = reader.read_u64::()?; + map.insert(id, FragmentationEntry::new(len, bytes, on_disk_bytes)); + } + + Ok(Self(map)) + } +} + +impl DroppedKvCallback for FragmentationMap { + fn on_dropped(&mut self, kv: &crate::InternalValue) { + if kv.key.value_type.is_indirection() { + let mut reader = &kv.value[..]; + + #[expect( + clippy::expect_used, + reason = "data is read and checked for corruption, so we expect to be able to deserialize BlobIndirection fine" + )] + let vptr = + BlobIndirection::decode_from(&mut reader).expect("should parse BlobIndirection"); + + let size = u64::from(vptr.size); + let on_disk_size = u64::from(vptr.vhandle.on_disk_size); + + self.0 + .entry(vptr.vhandle.blob_file_id) + .and_modify(|counter| { + counter.len += 1; + counter.bytes += size; + counter.on_disk_bytes += on_disk_size; + }) + .or_insert_with(|| FragmentationEntry { + bytes: size, + on_disk_bytes: on_disk_size, + len: 1, + }); + } + } +} + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::indexing_slicing)] +mod tests { + use super::*; + use crate::{ + coding::{Decode, Encode}, + compaction::stream::CompactionStream, + value::InternalValue, + vlog::ValueHandle, + ValueType, + }; + use std::collections::HashMap; + use test_log::test; + + #[test] + fn frag_map_merge_into() { + let mut map = FragmentationMap(HashMap::default()); + map.0.insert( + 0, + FragmentationEntry { + len: 1, + bytes: 1_000, + on_disk_bytes: 500, + }, + ); + map.0.insert( + 1, + FragmentationEntry { + len: 2, + bytes: 2_000, + on_disk_bytes: 1_000, + }, + ); + + // test merge_into + let mut diff = FragmentationMap(HashMap::default()); + diff.0.insert( + 0, + FragmentationEntry { + len: 3, + bytes: 3_000, + on_disk_bytes: 1_500, + }, + ); + diff.0.insert( + 3, + FragmentationEntry { + len: 4, + bytes: 4_000, + on_disk_bytes: 2_000, + }, + ); + + diff.merge_into(&mut map); + + assert_eq!(map.0.len(), 3); + assert_eq!(map.0[&0].len, 4); + assert_eq!(map.0[&0].bytes, 4_000); + assert_eq!(map.0[&0].on_disk_bytes, 2_000); + assert_eq!(map.0[&1].len, 2); + assert_eq!(map.0[&1].bytes, 2_000); + assert_eq!(map.0[&1].on_disk_bytes, 1_000); + assert_eq!(map.0[&3].len, 4); + assert_eq!(map.0[&3].bytes, 4_000); + assert_eq!(map.0[&3].on_disk_bytes, 2_000); + } + + #[test] + fn frag_map_roundtrip() { + let map = FragmentationMap({ + let mut map = HashMap::default(); + map.insert( + 0, + FragmentationEntry { + len: 1, + bytes: 1_000, + on_disk_bytes: 500, + }, + ); + map.insert( + 1, + FragmentationEntry { + len: 2, + bytes: 2_000, + on_disk_bytes: 1_000, + }, + ); + map + }); + + let encoded = map.encode_into_vec(); + let decoded = FragmentationMap::decode_from(&mut &encoded[..]).expect("should decode map"); + assert_eq!(map, decoded); + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_gc_count_drops() -> crate::Result<()> { + #[rustfmt::skip] + let vec = &[ + InternalValue::from_components("a", b"abc", 1, ValueType::Value), + + InternalValue::from_components("a", BlobIndirection { + size: 1000, + vhandle: ValueHandle { + blob_file_id: 0, + on_disk_size: 500, + offset: 0, + } + }.encode_into_vec(), 0, ValueType::Indirection), + ]; + + let mut my_watcher = FragmentationMap::default(); + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 1_000).with_drop_callback(&mut my_watcher); + + assert_eq!( + // TODO: Seqno is normally reset to 0 + InternalValue::from_components(*b"a", b"abc", 1, ValueType::Value), + iter.next().unwrap()?, + ); + + assert_eq!( + { + let mut map = HashMap::default(); + map.insert( + 0, + FragmentationEntry { + len: 1, + bytes: 1_000, + on_disk_bytes: 500, + }, + ); + map + }, + my_watcher.0, + ); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/blob_tree/handle.rs b/crates/lsm-tree/src/blob_tree/handle.rs new file mode 100644 index 000000000..364508482 --- /dev/null +++ b/crates/lsm-tree/src/blob_tree/handle.rs @@ -0,0 +1,32 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + coding::{Decode, Encode}, + vlog::ValueHandle, +}; +use std::io::{Read, Write}; +use varint_rs::{VarintReader, VarintWriter}; + +#[derive(Copy, Clone, Debug)] +pub struct BlobIndirection { + pub(crate) vhandle: ValueHandle, + pub(crate) size: u32, +} + +impl Encode for BlobIndirection { + fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error> { + self.vhandle.encode_into(writer)?; + writer.write_u32_varint(self.size)?; + Ok(()) + } +} + +impl Decode for BlobIndirection { + fn decode_from(reader: &mut R) -> Result { + let vhandle = ValueHandle::decode_from(reader)?; + let size = reader.read_u32_varint()?; + Ok(Self { vhandle, size }) + } +} diff --git a/crates/lsm-tree/src/blob_tree/ingest.rs b/crates/lsm-tree/src/blob_tree/ingest.rs new file mode 100644 index 000000000..92795c6bf --- /dev/null +++ b/crates/lsm-tree/src/blob_tree/ingest.rs @@ -0,0 +1,274 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + blob_tree::handle::BlobIndirection, file::BLOBS_FOLDER, table::Table, + tree::ingest::Ingestion as TableIngestion, vlog::BlobFileWriter, SeqNo, UserKey, UserValue, +}; + +/// Bulk ingestion for [`BlobTree`] +/// +/// Items NEED to be added in ascending key order. +/// +/// Uses table ingestion for the index and a blob file writer for large +/// values so both streams advance together. +pub struct BlobIngestion<'a> { + tree: &'a crate::BlobTree, + pub(crate) table: TableIngestion<'a>, + pub(crate) blob: BlobFileWriter, + seqno: SeqNo, + separation_threshold: u32, + last_key: Option, +} + +impl<'a> BlobIngestion<'a> { + /// Creates a new ingestion. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn new(tree: &'a crate::BlobTree) -> crate::Result { + #[expect( + clippy::expect_used, + reason = "cannot define blob tree without kv separation options" + )] + let kv = tree + .index + .config + .kv_separation_opts + .as_ref() + .expect("kv separation options should exist"); + + let blob_file_size = kv.file_target_size; + + let table = TableIngestion::new(&tree.index)?; + let blob = BlobFileWriter::new( + tree.index.0.blob_file_id_counter.clone(), + tree.index.config.path.join(BLOBS_FOLDER), + tree.index.id, + tree.index.config.descriptor_table.clone(), + )? + .use_target_size(blob_file_size) + .use_compression(kv.compression); + + let separation_threshold = kv.separation_threshold; + + Ok(Self { + tree, + table, + blob, + seqno: 0, + separation_threshold, + last_key: None, + }) + } + + /// Writes a key-value pair. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write(&mut self, key: UserKey, value: UserValue) -> crate::Result<()> { + // Check order before any blob I/O to avoid partial writes on failure + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + #[expect(clippy::cast_possible_truncation)] + let value_size = value.len() as u32; + + if value_size >= self.separation_threshold { + let vhandle = self.blob.write(&key, self.seqno, &value)?; + + let indirection = BlobIndirection { + vhandle, + size: value_size, + }; + + let cloned_key = key.clone(); + let res = self.table.write_indirection(key, indirection); + if res.is_ok() { + self.last_key = Some(cloned_key); + } + res + } else { + let cloned_key = key.clone(); + let res = self.table.write(key, value); + if res.is_ok() { + self.last_key = Some(cloned_key); + } + res + } + } + + /// Writes a tombstone for a key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write_tombstone(&mut self, key: UserKey) -> crate::Result<()> { + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + let cloned_key = key.clone(); + let res = self.table.write_tombstone(key); + if res.is_ok() { + self.last_key = Some(cloned_key); + } + res + } + + /// Writes a weak tombstone for a key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write_weak_tombstone(&mut self, key: UserKey) -> crate::Result<()> { + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + let cloned_key = key.clone(); + let res = self.table.write_weak_tombstone(key); + if res.is_ok() { + self.last_key = Some(cloned_key); + } + res + } + + /// Finishes the ingestion. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[allow(clippy::significant_drop_tightening)] + pub fn finish(self) -> crate::Result<()> { + use crate::AbstractTree; + + let index = self.index().clone(); + + // CRITICAL SECTION: Atomic flush + seqno allocation + registration + // + // For BlobTree, we must coordinate THREE components atomically: + // 1. Index tree memtable flush + // 2. Value log blob files + // 3. Index tree tables (with blob indirections) + // + // The sequence ensures all components see the same global_seqno: + // 1. Acquire flush lock on index tree + // 2. Flush index tree active memtable + // 3. Finalize blob writer (creates blob files) + // 4. Finalize table writer (creates index tables) + // 5. Allocate next global seqno + // 6. Recover tables with that seqno + // 7. Register version with same seqno + blob files + // + // This prevents race conditions where blob files and their index + // entries could have mismatched sequence numbers. + let flush_lock = index.get_flush_lock(); + + // Flush any pending index memtable writes to ensure ingestion sees + // a consistent snapshot of the index. + // We call rotate + flush directly because we already hold the lock. + index.rotate_memtable(); + index.flush(&flush_lock, 0)?; + + // Finalize the blob writer first, ensuring all large values are + // written to blob files before we finalize the index tables that + // reference them. + let blob_files = self.blob.finish()?; + + // Finalize the table writer, creating index tables with blob + // indirections pointing to the blob files we just created. + let results = self.table.writer.finish()?; + + // Acquire locks for version registration on the index tree. We must + // hold both the compaction state lock and version history lock to + // safely modify the tree's version. + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut _compaction_state = index.compaction_state.lock().expect("lock is poisoned"); + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_lock = index.version_history.write().expect("lock is poisoned"); + + // Allocate the next global sequence number. This seqno will be shared + // by all ingested tables, blob files, and the version that registers + // them, ensuring consistent MVCC snapshots across the value log. + let global_seqno = index.config.seqno.next(); + + // Recover all created index tables, assigning them the global_seqno + // we just allocated. These tables contain indirections to the blob + // files created above, so they must share the same sequence number + // for MVCC correctness. + // + // We intentionally do NOT pin filter/index blocks here. Large ingests + // are typically placed in level 1, and pinning would increase memory + // pressure unnecessarily. + let created_tables = results + .into_iter() + .map(|(table_id, checksum)| -> crate::Result
{ + Table::recover( + index + .config + .path + .join(crate::file::TABLES_FOLDER) + .join(table_id.to_string()), + checksum, + global_seqno, + index.id, + index.config.cache.clone(), + index.config.descriptor_table.clone(), + false, + false, + #[cfg(feature = "metrics")] + index.metrics.clone(), + ) + }) + .collect::>>()?; + + // Upgrade the version with our ingested tables and blob files, using + // the global_seqno we allocated earlier. This ensures the version, + // tables, and blob files all share the same sequence number, which is + // critical for GC correctness - we must not delete blob files that are + // still referenced by visible snapshots. + // + // We use upgrade_version_with_seqno (instead of upgrade_version) because + // we need precise control over the seqno: it must match the seqno we + // already assigned to the recovered tables. + version_lock.upgrade_version_with_seqno( + &index.config.path, + |current| { + let mut copy = current.clone(); + copy.version = + copy.version + .with_new_l0_run(&created_tables, Some(&blob_files), None); + Ok(copy) + }, + global_seqno, + &self.tree.index.config.visible_seqno, + )?; + + // Perform maintenance on the version history (e.g., clean up old versions). + // We use gc_watermark=0 since ingestion doesn't affect sealed memtables. + if let Err(e) = version_lock.maintenance(&index.config.path, 0) { + log::warn!("Version GC failed: {e:?}"); + } + + Ok(()) + } + + #[inline] + fn index(&self) -> &crate::Tree { + &self.tree.index + } +} diff --git a/crates/lsm-tree/src/blob_tree/mod.rs b/crates/lsm-tree/src/blob_tree/mod.rs new file mode 100644 index 000000000..d38e528c7 --- /dev/null +++ b/crates/lsm-tree/src/blob_tree/mod.rs @@ -0,0 +1,631 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod gc; +pub mod handle; +pub mod ingest; + +#[doc(hidden)] +pub use gc::{FragmentationEntry, FragmentationMap}; + +use crate::{ + abstract_tree::{AbstractTree, RangeItem}, + coding::Decode, + iter_guard::{IterGuard, IterGuardImpl}, + table::Table, + tree::inner::MemtableId, + value::InternalValue, + version::Version, + vlog::{Accessor, BlobFile, BlobFileWriter}, + Cache, Config, Memtable, SeqNo, TableId, TreeId, UserKey, UserValue, +}; +use handle::BlobIndirection; +use std::{ + ops::RangeBounds, + path::{Path, PathBuf}, + sync::{Arc, MutexGuard}, +}; + +/// Iterator value guard +pub struct Guard { + tree: crate::BlobTree, + version: Version, + kv: crate::Result, +} + +impl IterGuard for Guard { + fn into_inner_if( + self, + pred: impl Fn(&UserKey) -> bool, + ) -> crate::Result<(UserKey, Option)> { + let kv = self.kv?; + + if pred(&kv.key.user_key) { + resolve_value_handle( + self.tree.id(), + self.tree.blobs_folder.as_path(), + &self.tree.index.config.cache, + &self.version, + kv, + ) + .map(|(k, v)| (k, Some(v))) + } else { + Ok((kv.key.user_key, None)) + } + } + + fn key(self) -> crate::Result { + self.kv.map(|kv| kv.key.user_key) + } + + fn size(self) -> crate::Result { + let kv = self.kv?; + + if kv.key.value_type.is_indirection() { + let mut cursor = std::io::Cursor::new(kv.value); + Ok(BlobIndirection::decode_from(&mut cursor)?.size) + } else { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 max length")] + Ok(kv.value.len() as u32) + } + } + + fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { + resolve_value_handle( + self.tree.id(), + self.tree.blobs_folder.as_path(), + &self.tree.index.config.cache, + &self.version, + self.kv?, + ) + } +} + +fn resolve_value_handle( + tree_id: TreeId, + blobs_folder: &Path, + cache: &Cache, + version: &Version, + item: InternalValue, +) -> RangeItem { + if item.key.value_type.is_indirection() { + let mut cursor = std::io::Cursor::new(item.value); + let vptr = BlobIndirection::decode_from(&mut cursor)?; + + // Resolve indirection using value log + match Accessor::new(&version.blob_files).get( + tree_id, + blobs_folder, + &item.key.user_key, + &vptr.vhandle, + cache, + ) { + Ok(Some(v)) => { + let k = item.key.user_key; + Ok((k, v)) + } + Ok(None) => { + panic!( + "value handle ({:?} => {:?}) did not match any blob - this is a bug; version={}", + item.key.user_key, vptr.vhandle, + version.id(), + ); + } + Err(e) => Err(e), + } + } else { + let k = item.key.user_key; + let v = item.value; + Ok((k, v)) + } +} + +/// A key-value-separated log-structured merge tree +/// +/// This tree is a composite structure, consisting of an +/// index tree (LSM-tree) and a log-structured value log +/// to reduce write amplification. +#[derive(Clone)] +pub struct BlobTree { + /// Index tree that holds value handles or small inline values + #[doc(hidden)] + pub index: crate::Tree, + + blobs_folder: Arc, +} + +impl BlobTree { + pub(crate) fn open(config: Config) -> crate::Result { + use crate::file::{fsync_directory, BLOBS_FOLDER}; + + let index = crate::Tree::open(config)?; + + let blobs_folder = index.config.path.join(BLOBS_FOLDER); + std::fs::create_dir_all(&blobs_folder)?; + fsync_directory(&blobs_folder)?; + + let blob_file_id_to_continue_with = index + .current_version() + .blob_files + .list_ids() + .max() + .map(|x| x + 1) + .unwrap_or_default(); + + index + .0 + .blob_file_id_counter + .set(blob_file_id_to_continue_with); + + Ok(Self { + index, + blobs_folder: Arc::new(blobs_folder), + }) + } +} + +impl AbstractTree for BlobTree { + fn print_trace(&self, key: &[u8]) -> crate::Result<()> { + self.index.print_trace(key) + } + + fn table_file_cache_size(&self) -> usize { + self.index.table_file_cache_size() + } + + fn get_version_history_lock( + &self, + ) -> std::sync::RwLockWriteGuard<'_, crate::version::SuperVersions> { + self.index.get_version_history_lock() + } + + fn next_table_id(&self) -> TableId { + self.index.next_table_id() + } + + fn id(&self) -> crate::TreeId { + self.index.id() + } + + fn get_internal_entry(&self, key: &[u8], seqno: SeqNo) -> crate::Result> { + self.index.get_internal_entry(key, seqno) + } + + fn current_version(&self) -> Version { + self.index.current_version() + } + + #[cfg(feature = "metrics")] + fn metrics(&self) -> &Arc { + self.index.metrics() + } + + fn version_free_list_len(&self) -> usize { + self.index.version_free_list_len() + } + + fn prefix>( + &self, + prefix: K, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static> { + use crate::range::prefix_to_range; + + let super_version = self.index.get_version_for_snapshot(seqno); + let tree = self.clone(); + + let range = prefix_to_range(prefix.as_ref()); + + Box::new( + crate::Tree::create_internal_range(super_version.clone(), &range, seqno, index).map( + move |kv| { + IterGuardImpl::Blob(Guard { + tree: tree.clone(), + version: super_version.version.clone(), + kv, + }) + }, + ), + ) + } + + fn range, R: RangeBounds>( + &self, + range: R, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static> { + let super_version = self.index.get_version_for_snapshot(seqno); + let tree = self.clone(); + + Box::new( + crate::Tree::create_internal_range(super_version.clone(), &range, seqno, index).map( + move |kv| { + IterGuardImpl::Blob(Guard { + tree: tree.clone(), + version: super_version.version.clone(), + kv, + }) + }, + ), + ) + } + + fn tombstone_count(&self) -> u64 { + self.index.tombstone_count() + } + + fn weak_tombstone_count(&self) -> u64 { + self.index.weak_tombstone_count() + } + + fn weak_tombstone_reclaimable_count(&self) -> u64 { + self.index.weak_tombstone_reclaimable_count() + } + + fn drop_range, R: RangeBounds>(&self, range: R) -> crate::Result<()> { + self.index.drop_range(range) + } + + fn clear(&self) -> crate::Result<()> { + let config = self.tree_config(); + let mut versions = self.get_version_history_lock(); + + versions.upgrade_version( + &config.path, + |v| { + let mut copy = v.clone(); + copy.active_memtable = + Arc::new(Memtable::new(self.index.memtable_id_counter.next())); + copy.sealed_memtables = Arc::default(); + copy.version = Version::new(v.version.id() + 1, self.tree_type()); + Ok(copy) + }, + &config.seqno, + &config.visible_seqno, + ) + } + + fn major_compact(&self, target_size: u64, seqno_threshold: SeqNo) -> crate::Result<()> { + self.index.major_compact(target_size, seqno_threshold) + } + + fn clear_active_memtable(&self) { + self.index.clear_active_memtable(); + } + + fn l0_run_count(&self) -> usize { + self.index.l0_run_count() + } + + fn blob_file_count(&self) -> usize { + self.current_version().blob_file_count() + } + + // NOTE: We skip reading from the value log + // because the vHandles already store the value size + fn size_of>(&self, key: K, seqno: SeqNo) -> crate::Result> { + let Some(item) = self.index.get_internal_entry(key.as_ref(), seqno)? else { + return Ok(None); + }; + + Ok(Some(if item.key.value_type.is_indirection() { + let mut cursor = std::io::Cursor::new(item.value); + let vptr = BlobIndirection::decode_from(&mut cursor)?; + vptr.size + } else { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + { + item.value.len() as u32 + } + })) + } + + fn stale_blob_bytes(&self) -> u64 { + self.current_version().gc_stats().stale_bytes() + } + + fn filter_size(&self) -> u64 { + self.index.filter_size() + } + + fn pinned_filter_size(&self) -> usize { + self.index.pinned_filter_size() + } + + fn pinned_block_index_size(&self) -> usize { + self.index.pinned_block_index_size() + } + + fn sealed_memtable_count(&self) -> usize { + self.index.sealed_memtable_count() + } + + fn get_flush_lock(&self) -> MutexGuard<'_, ()> { + self.index.get_flush_lock() + } + + fn flush_to_tables( + &self, + stream: impl Iterator>, + ) -> crate::Result, Option>)>> { + use crate::{ + coding::Encode, file::BLOBS_FOLDER, file::TABLES_FOLDER, + table::multi_writer::MultiWriter, + }; + + let start = std::time::Instant::now(); + + let table_folder = self.index.config.path.join(TABLES_FOLDER); + + let data_block_size = self.index.config.data_block_size_policy.get(0); + + let data_block_restart_interval = + self.index.config.data_block_restart_interval_policy.get(0); + let index_block_restart_interval = + self.index.config.index_block_restart_interval_policy.get(0); + + let data_block_compression = self.index.config.data_block_compression_policy.get(0); + let index_block_compression = self.index.config.index_block_compression_policy.get(0); + + let data_block_hash_ratio = self.index.config.data_block_hash_ratio_policy.get(0); + + let index_partitioning = self.index.config.index_block_partitioning_policy.get(0); + let filter_partitioning = self.index.config.filter_block_partitioning_policy.get(0); + + log::debug!("Flushing memtable(s) and performing key-value separation, data_block_restart_interval={data_block_restart_interval}, index_block_restart_interval={index_block_restart_interval}, data_block_size={data_block_size}, data_block_compression={data_block_compression:?}, index_block_compression={index_block_compression:?}"); + log::debug!("=> to table(s) in {}", table_folder.display()); + log::debug!("=> to blob file(s) at {}", self.blobs_folder.display()); + + let mut table_writer = MultiWriter::new( + table_folder.clone(), + self.index.table_id_counter.clone(), + 64 * 1_024 * 1_024, + 0, + )? + .use_data_block_restart_interval(data_block_restart_interval) + .use_index_block_restart_interval(index_block_restart_interval) + .use_data_block_compression(data_block_compression) + .use_index_block_compression(index_block_compression) + .use_data_block_size(data_block_size) + .use_data_block_hash_ratio(data_block_hash_ratio) + .use_bloom_policy({ + use crate::config::FilterPolicyEntry::{Bloom, None}; + use crate::table::filter::BloomConstructionPolicy; + + match self.index.config.filter_policy.get(0) { + Bloom(policy) => policy, + None => BloomConstructionPolicy::BitsPerKey(0.0), + } + }); + + if index_partitioning { + table_writer = table_writer.use_partitioned_index(); + } + if filter_partitioning { + table_writer = table_writer.use_partitioned_filter(); + } + + #[expect( + clippy::expect_used, + reason = "cannot create blob tree without defining kv separation options" + )] + let kv_opts = self + .index + .config + .kv_separation_opts + .as_ref() + .expect("kv separation options should exist"); + + let mut blob_writer = BlobFileWriter::new( + self.index.0.blob_file_id_counter.clone(), + self.index.config.path.join(BLOBS_FOLDER), + self.id(), + self.index.config.descriptor_table.clone(), + )? + .use_target_size(kv_opts.file_target_size) + .use_compression(kv_opts.compression); + + let separation_threshold = kv_opts.separation_threshold; + + for item in stream { + let item = item?; + + if item.is_tombstone() { + // NOTE: Still need to add tombstone to index tree + // But no blob to blob writer + table_writer.write(InternalValue::new(item.key, UserValue::empty()))?; + continue; + } + + let value = item.value; + + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + let value_size = value.len() as u32; + + if value_size >= separation_threshold { + let vhandle = blob_writer.write(&item.key.user_key, item.key.seqno, &value)?; + + let indirection = BlobIndirection { + vhandle, + size: value_size, + }; + + table_writer.write({ + let mut vptr = + InternalValue::new(item.key.clone(), indirection.encode_into_vec()); + vptr.key.value_type = crate::ValueType::Indirection; + vptr + })?; + + table_writer.register_blob(indirection); + } else { + table_writer.write(InternalValue::new(item.key, value))?; + } + } + + let blob_files = blob_writer.finish()?; + + let result = table_writer.finish()?; + + log::debug!("Flushed memtable(s) in {:?}", start.elapsed()); + + let pin_filter = self.index.config.filter_block_pinning_policy.get(0); + let pin_index = self.index.config.index_block_pinning_policy.get(0); + + // Load tables + let tables = result + .into_iter() + .map(|(table_id, checksum)| -> crate::Result
{ + Table::recover( + table_folder.join(table_id.to_string()), + checksum, + 0, + self.index.id, + self.index.config.cache.clone(), + self.index.config.descriptor_table.clone(), + pin_filter, + pin_index, + #[cfg(feature = "metrics")] + self.index.metrics.clone(), + ) + }) + .collect::>>()?; + + Ok(Some((tables, Some(blob_files)))) + } + + fn register_tables( + &self, + tables: &[Table], + blob_files: Option<&[BlobFile]>, + frag_map: Option, + sealed_memtables_to_delete: &[MemtableId], + gc_watermark: SeqNo, + ) -> crate::Result<()> { + self.index.register_tables( + tables, + blob_files, + frag_map, + sealed_memtables_to_delete, + gc_watermark, + ) + } + + fn compact( + &self, + strategy: Arc, + seqno_threshold: SeqNo, + ) -> crate::Result<()> { + self.index.compact(strategy, seqno_threshold) + } + + fn get_next_table_id(&self) -> TableId { + self.index.get_next_table_id() + } + + fn tree_config(&self) -> &Config { + &self.index.config + } + + fn get_highest_seqno(&self) -> Option { + self.index.get_highest_seqno() + } + + fn active_memtable(&self) -> Arc { + self.index.active_memtable() + } + + fn rotate_memtable(&self) -> Option> { + self.index.rotate_memtable() + } + + fn table_count(&self) -> usize { + self.index.table_count() + } + + fn level_table_count(&self, idx: usize) -> Option { + self.index.level_table_count(idx) + } + + fn approximate_len(&self) -> usize { + self.index.approximate_len() + } + + // NOTE: Override the default implementation to not fetch + // data from the value log, so we get much faster key reads + fn is_empty(&self, seqno: SeqNo, index: Option<(Arc, SeqNo)>) -> crate::Result { + self.index.is_empty(seqno, index) + } + + // NOTE: Override the default implementation to not fetch + // data from the value log, so we get much faster key reads + fn contains_key>(&self, key: K, seqno: SeqNo) -> crate::Result { + self.index.contains_key(key, seqno) + } + + // NOTE: Override the default implementation to not fetch + // data from the value log, so we get much faster scans + fn len(&self, seqno: SeqNo, index: Option<(Arc, SeqNo)>) -> crate::Result { + self.index.len(seqno, index) + } + + fn disk_space(&self) -> u64 { + let version = self.current_version(); + self.index.disk_space() + version.blob_files.on_disk_size() + } + + fn get_highest_memtable_seqno(&self) -> Option { + self.index.get_highest_memtable_seqno() + } + + fn get_highest_persisted_seqno(&self) -> Option { + self.index.get_highest_persisted_seqno() + } + + fn insert, V: Into>( + &self, + key: K, + value: V, + seqno: SeqNo, + ) -> (u64, u64) { + self.index.insert(key, value.into(), seqno) + } + + fn get>(&self, key: K, seqno: SeqNo) -> crate::Result> { + let key = key.as_ref(); + + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let super_version = self + .index + .version_history + .read() + .expect("lock is poisoned") + .get_version_for_snapshot(seqno); + + let Some(item) = crate::Tree::get_internal_entry_from_version(&super_version, key, seqno)? + else { + return Ok(None); + }; + + let (_, v) = resolve_value_handle( + self.id(), + self.blobs_folder.as_path(), + &self.index.config.cache, + &super_version.version, + item, + )?; + + Ok(Some(v)) + } + + fn remove>(&self, key: K, seqno: SeqNo) -> (u64, u64) { + self.index.remove(key, seqno) + } + + fn remove_weak>(&self, key: K, seqno: SeqNo) -> (u64, u64) { + self.index.remove_weak(key, seqno) + } +} diff --git a/crates/lsm-tree/src/cache.rs b/crates/lsm-tree/src/cache.rs new file mode 100644 index 000000000..465c80c8c --- /dev/null +++ b/crates/lsm-tree/src/cache.rs @@ -0,0 +1,161 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::table::block::Header; +use crate::table::{Block, BlockOffset}; +use crate::{GlobalTableId, UserValue}; +use quick_cache::sync::Cache as QuickCache; +use quick_cache::Weighter; + +const TAG_BLOCK: u8 = 0; +const TAG_BLOB: u8 = 1; + +#[derive(Clone)] +enum Item { + Block(Block), + Blob(UserValue), +} + +#[derive(Eq, std::hash::Hash, PartialEq)] +struct CacheKey(u8, u64, u64, u64); + +impl From<(u8, u64, u64, u64)> for CacheKey { + fn from((tag, root_id, table_id, offset): (u8, u64, u64, u64)) -> Self { + Self(tag, root_id, table_id, offset) + } +} + +#[derive(Clone)] +struct BlockWeighter; + +impl Weighter for BlockWeighter { + fn weight(&self, _: &CacheKey, item: &Item) -> u64 { + use Item::{Blob, Block}; + + match item { + Block(b) => (Header::serialized_len() as u64) + u64::from(b.header.uncompressed_length), + Blob(b) => b.len() as u64, + } + } +} + +/// Cache, in which blocks or blobs are cached in-memory +/// after being retrieved from disk +/// +/// This speeds up consecutive queries to nearby data, improving +/// read performance for hot data. +/// +/// # Examples +/// +/// Sharing cache between multiple trees +/// +/// ``` +/// # use lsm_tree::{Tree, Config, Cache}; +/// # use std::sync::Arc; +/// # +/// // Provide 64 MB of cache capacity +/// let cache = Arc::new(Cache::with_capacity_bytes(64 * 1_000 * 1_000)); +/// +/// # let folder = tempfile::tempdir()?; +/// let tree1 = Config::new(folder, Default::default(), Default::default()).use_cache(cache.clone()).open()?; +/// # let folder = tempfile::tempdir()?; +/// let tree2 = Config::new(folder, Default::default(), Default::default()).use_cache(cache.clone()).open()?; +/// # +/// # Ok::<(), lsm_tree::Error>(()) +/// ``` +pub struct Cache { + // NOTE: rustc_hash performed best: https://fjall-rs.github.io/post/fjall-2-1 + /// Concurrent cache implementation + data: QuickCache, + + /// Capacity in bytes + capacity: u64, +} + +impl Cache { + /// Creates a new block cache with roughly `n` bytes of capacity. + #[must_use] + pub fn with_capacity_bytes(bytes: u64) -> Self { + use quick_cache::sync::DefaultLifecycle; + + #[expect(clippy::expect_used, reason = "nothing we can do if it fails")] + let opts = quick_cache::OptionsBuilder::new() + .weight_capacity(bytes) + .hot_allocation(0.8) + .estimated_items_capacity(10_000) + .build() + .expect("cache options should be valid"); + + let quick_cache = QuickCache::with_options( + opts, + BlockWeighter, + rustc_hash::FxBuildHasher, + DefaultLifecycle::default(), + ); + + Self { + data: quick_cache, + capacity: bytes, + } + } + + /// Returns the amount of cached bytes. + #[must_use] + pub fn size(&self) -> u64 { + self.data.weight() + } + + /// Returns the cache capacity in bytes. + #[must_use] + pub fn capacity(&self) -> u64 { + self.capacity + } + + #[doc(hidden)] + #[must_use] + pub fn get_block(&self, id: GlobalTableId, offset: BlockOffset) -> Option { + let key: CacheKey = (TAG_BLOCK, id.tree_id(), id.table_id(), *offset).into(); + + Some(match self.data.get(&key)? { + Item::Block(block) => block, + Item::Blob(_) => unreachable!("invalid cache item"), + }) + } + + #[doc(hidden)] + pub fn insert_block(&self, id: GlobalTableId, offset: BlockOffset, block: Block) { + self.data.insert( + (TAG_BLOCK, id.tree_id(), id.table_id(), *offset).into(), + Item::Block(block), + ); + } + + #[doc(hidden)] + pub fn insert_blob( + &self, + vlog_id: crate::TreeId, + vhandle: &crate::vlog::ValueHandle, + value: UserValue, + ) { + self.data.insert( + (TAG_BLOB, vlog_id, vhandle.blob_file_id, vhandle.offset).into(), + Item::Blob(value), + ); + } + + #[doc(hidden)] + #[must_use] + pub fn get_blob( + &self, + vlog_id: crate::TreeId, + vhandle: &crate::vlog::ValueHandle, + ) -> Option { + let key: CacheKey = (TAG_BLOB, vlog_id, vhandle.blob_file_id, vhandle.offset).into(); + + Some(match self.data.get(&key)? { + Item::Blob(blob) => blob, + Item::Block(_) => unreachable!("invalid cache item"), + }) + } +} diff --git a/crates/lsm-tree/src/checksum.rs b/crates/lsm-tree/src/checksum.rs new file mode 100644 index 000000000..3e0f88ff6 --- /dev/null +++ b/crates/lsm-tree/src/checksum.rs @@ -0,0 +1,96 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum ChecksumType { + Xxh3, +} + +impl From for u8 { + fn from(val: ChecksumType) -> Self { + match val { + ChecksumType::Xxh3 => 0, + } + } +} + +/// An 128-bit checksum +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct Checksum(u128); + +impl From for Checksum { + fn from(value: sfa::Checksum) -> Self { + Self(value.into_u128()) + } +} + +impl std::fmt::Display for Checksum { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl Checksum { + /// Wraps a checksum value. + #[must_use] + pub fn from_raw(value: u128) -> Self { + Self(value) + } + + /// Returns the raw 128-bit integer. + #[must_use] + pub fn into_u128(self) -> u128 { + self.0 + } + + pub(crate) fn check(&self, expected: Self) -> crate::Result<()> { + if self.0 == expected.0 { + Ok(()) + } else { + Err(crate::Error::ChecksumMismatch { + expected, + got: *self, + }) + } + } +} + +pub struct ChecksummedWriter { + inner: W, + hasher: xxhash_rust::xxh3::Xxh3Default, +} + +impl std::io::Seek for ChecksummedWriter { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} + +impl ChecksummedWriter { + pub fn new(writer: W) -> Self { + Self { + inner: writer, + hasher: xxhash_rust::xxh3::Xxh3Default::new(), + } + } + + pub fn checksum(&self) -> Checksum { + Checksum::from_raw(self.hasher.digest128()) + } + + pub fn inner_mut(&mut self) -> &mut W { + &mut self.inner + } +} + +impl std::io::Write for ChecksummedWriter { + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } + + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.hasher.update(buf); + self.inner.write(buf) + } +} diff --git a/crates/lsm-tree/src/coding.rs b/crates/lsm-tree/src/coding.rs new file mode 100644 index 000000000..862347ae7 --- /dev/null +++ b/crates/lsm-tree/src/coding.rs @@ -0,0 +1,30 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::io::{Read, Write}; + +/// Trait to serialize stuff +pub trait Encode { + /// Serializes into writer. + fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error>; + + /// Serializes into vector. + fn encode_into_vec(&self) -> Vec { + let mut v = vec![]; + #[expect( + clippy::expect_used, + reason = "encoding into a vec is not expected to fail" + )] + self.encode_into(&mut v).expect("cannot fail"); + v + } +} + +/// Trait to deserialize stuff +pub trait Decode { + /// Deserializes from reader. + fn decode_from(reader: &mut R) -> Result + where + Self: Sized; +} diff --git a/crates/lsm-tree/src/compaction/drop_range.rs b/crates/lsm-tree/src/compaction/drop_range.rs new file mode 100644 index 000000000..4134a195b --- /dev/null +++ b/crates/lsm-tree/src/compaction/drop_range.rs @@ -0,0 +1,100 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy}; +use crate::compaction::state::CompactionState; +use crate::version::Version; +use crate::{config::Config, slice::Slice, version::run::Ranged, KeyRange}; +use crate::{HashSet, Table}; +use std::ops::{Bound, RangeBounds}; + +#[derive(Clone, Debug)] +pub struct OwnedBounds { + pub start: Bound, + pub end: Bound, +} + +impl RangeBounds for OwnedBounds { + fn start_bound(&self) -> Bound<&Slice> { + match &self.start { + Bound::Unbounded => Bound::Unbounded, + Bound::Included(key) => Bound::Included(key), + Bound::Excluded(key) => Bound::Excluded(key), + } + } + + fn end_bound(&self) -> Bound<&Slice> { + match &self.end { + Bound::Unbounded => Bound::Unbounded, + Bound::Included(key) => Bound::Included(key), + Bound::Excluded(key) => Bound::Excluded(key), + } + } +} + +impl OwnedBounds { + #[must_use] + pub fn contains(&self, range: &KeyRange) -> bool { + let lower_ok = match &self.start { + Bound::Unbounded => true, + Bound::Included(key) => key.as_ref() <= range.min().as_ref(), + Bound::Excluded(key) => key.as_ref() < range.min().as_ref(), + }; + + if !lower_ok { + return false; + } + + match &self.end { + Bound::Unbounded => true, + Bound::Included(key) => key.as_ref() >= range.max().as_ref(), + Bound::Excluded(key) => key.as_ref() > range.max().as_ref(), + } + } +} + +/// Drops all tables that are **contained** in a key range +pub struct Strategy { + bounds: OwnedBounds, +} + +impl Strategy { + /// Configures a new `DropRange` compaction strategy. + #[must_use] + pub fn new(bounds: OwnedBounds) -> Self { + Self { bounds } + } +} + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + "DropRangeCompaction" + } + + fn choose(&self, version: &Version, _: &Config, state: &CompactionState) -> Choice { + let table_ids: HashSet<_> = version + .iter_levels() + .flat_map(|lvl| lvl.iter()) + .flat_map(|run| { + run.range_overlap_indexes(&self.bounds) + .and_then(|(lo, hi)| run.get(lo..=hi)) + .unwrap_or_default() + .iter() + .filter(|x| self.bounds.contains(x.key_range())) + }) + .map(Table::id) + .collect(); + + // NOTE: This should generally not occur because of the + // tree-level major compaction lock + // But just as a fail-safe... + let some_hidden = table_ids.iter().any(|&id| state.hidden_set().is_hidden(id)); + + if some_hidden { + Choice::DoNothing + } else { + Choice::Drop(table_ids) + } + } +} diff --git a/crates/lsm-tree/src/compaction/fifo.rs b/crates/lsm-tree/src/compaction/fifo.rs new file mode 100644 index 000000000..7c2a17fd1 --- /dev/null +++ b/crates/lsm-tree/src/compaction/fifo.rs @@ -0,0 +1,310 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy}; +use crate::{ + compaction::state::CompactionState, config::Config, time::unix_timestamp, version::Version, + HashSet, KvPair, +}; + +#[doc(hidden)] +pub const NAME: &str = "FifoCompaction"; + +/// FIFO-style compaction +/// +/// Limits the tree size to roughly `limit` bytes, deleting the oldest table(s) +/// when the threshold is reached. +/// +/// Will also merge tables if the number of tables in level 0 grows too much, which +/// could cause write stalls. +/// +/// Additionally, a (lazy) TTL can be configured to drop old tables. +/// +/// ###### Caution +/// +/// Only use it for specific workloads where: +/// +/// 1) You only want to store recent data (unimportant logs, ...) +/// 2) The key order of inserts is strictly monotonically increasing or decreasing +/// 3) You only insert new data (no updates/deletes) +#[derive(Clone)] +pub struct Strategy { + /// Data set size limit in bytes + pub limit: u64, + + /// TTL in seconds, will be disabled if 0 or None + pub ttl_seconds: Option, +} + +impl Strategy { + /// Configures a new `Fifo` compaction strategy + #[must_use] + pub fn new(limit: u64, ttl_seconds: Option) -> Self { + Self { limit, ttl_seconds } + } +} + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + NAME + } + + fn get_config(&self) -> Vec { + vec![ + ( + crate::UserKey::from("fifo_limit"), + crate::UserValue::from(self.limit.to_le_bytes()), + ), + ( + crate::UserKey::from("fifo_ttl"), + crate::UserValue::from(if self.ttl_seconds.is_some() { + [1u8] + } else { + [0u8] + }), + ), + ( + crate::UserKey::from("fifo_ttl_seconds"), + crate::UserValue::from(self.ttl_seconds.map(u64::to_le_bytes).unwrap_or_default()), + ), + ] + } + + fn choose(&self, version: &Version, _: &Config, state: &CompactionState) -> Choice { + let first_level = version.l0(); + + // Early return avoids unnecessary work and keeps FIFO a no-op when there is nothing to do. + if first_level.is_empty() { + return Choice::DoNothing; + } + + assert!(first_level.is_disjoint(), "L0 needs to be disjoint"); + + assert!( + !version.level_is_busy(0, state.hidden_set()), + "FIFO compaction never compacts", + ); + + // Account for both table file bytes and value-log (blob) bytes to enforce the true space limit. + let db_size = first_level.size() + version.blob_files.on_disk_size(); + + let mut ids_to_drop = HashSet::default(); + + // Compute TTL cutoff once and perform a single pass to mark expired tables and + // accumulate their sizes. Also collect non-expired tables for possible size-based drops. + let ttl_cutoff = match self.ttl_seconds { + Some(s) if s > 0 => Some( + unix_timestamp() + .as_nanos() + .saturating_sub(u128::from(s) * 1_000_000_000u128), + ), + _ => None, + }; + + let mut ttl_dropped_bytes = 0u64; + let mut alive = Vec::new(); + + for table in first_level.iter().flat_map(|run| run.iter()) { + let expired = + ttl_cutoff.is_some_and(|cutoff| u128::from(table.metadata.created_at) <= cutoff); + + if expired { + ids_to_drop.insert(table.id()); + let linked_blob_file_bytes = table.referenced_blob_bytes().unwrap_or_default(); + ttl_dropped_bytes += table.file_size() + linked_blob_file_bytes; + } else { + alive.push(table); + } + } + + // Subtract TTL-selected bytes to see if we're still over the limit. + let size_after_ttl = db_size.saturating_sub(ttl_dropped_bytes); + + // If we still exceed the limit, drop additional oldest tables until within the limit. + if size_after_ttl > self.limit { + let overshoot = size_after_ttl - self.limit; + + let mut collected_bytes = 0; + + // Oldest-first list by creation time from the non-expired set. + alive.sort_by_key(|t| t.metadata.created_at); + + for table in alive { + if collected_bytes >= overshoot { + break; + } + + ids_to_drop.insert(table.id()); + + let linked_blob_file_bytes = table.referenced_blob_bytes().unwrap_or_default(); + collected_bytes += table.file_size() + linked_blob_file_bytes; + } + } + + if ids_to_drop.is_empty() { + Choice::DoNothing + } else { + Choice::Drop(ids_to_drop) + } + } +} + +#[cfg(test)] +mod tests { + use super::Strategy; + use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter}; + use std::sync::Arc; + + #[test] + fn fifo_empty_levels() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let fifo = Arc::new(Strategy::new(1, None)); + tree.compact(fifo, 0)?; + + assert_eq!(0, tree.table_count()); + Ok(()) + } + + #[test] + fn fifo_below_limit() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..4u8 { + tree.insert([b'k', i].as_slice(), "v", u64::from(i)); + tree.flush_active_memtable(u64::from(i))?; + } + + let before = tree.table_count(); + let fifo = Arc::new(Strategy::new(u64::MAX, None)); + tree.compact(fifo, 4)?; + + assert_eq!(before, tree.table_count()); + Ok(()) + } + + #[test] + fn fifo_more_than_limit() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..4u8 { + tree.insert([b'k', i].as_slice(), "v", u64::from(i)); + tree.flush_active_memtable(u64::from(i))?; + } + + let before = tree.table_count(); + // Very small limit forces dropping oldest tables + let fifo = Arc::new(Strategy::new(1, None)); + tree.compact(fifo, 4)?; + + assert!(tree.table_count() < before); + Ok(()) + } + + #[test] + fn fifo_more_than_limit_blobs() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + for i in 0..3u8 { + tree.insert([b'k', i].as_slice(), "$", u64::from(i)); + tree.flush_active_memtable(u64::from(i))?; + } + + let before = tree.table_count(); + let fifo = Arc::new(Strategy::new(1, None)); + tree.compact(fifo, 3)?; + + assert!(tree.table_count() < before); + Ok(()) + } + + #[test] + fn fifo_ttl() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Freeze time and create first (older) table at t=1000s + crate::time::set_unix_timestamp_for_test(Some(std::time::Duration::from_secs(1_000))); + tree.insert("a", "1", 0); + tree.flush_active_memtable(0)?; + + // Advance time and create second (newer) table at t=1005s + crate::time::set_unix_timestamp_for_test(Some(std::time::Duration::from_secs(1_005))); + tree.insert("b", "2", 1); + tree.flush_active_memtable(1)?; + + // Now set current time to t=1011s; with TTL=10s, cutoff=1001s => drop first only + crate::time::set_unix_timestamp_for_test(Some(std::time::Duration::from_secs(1_011))); + + assert_eq!(2, tree.table_count()); + + let fifo = Arc::new(Strategy::new(u64::MAX, Some(10))); + tree.compact(fifo, 2)?; + + assert_eq!(1, tree.table_count()); + + // Reset override + crate::time::set_unix_timestamp_for_test(None); + Ok(()) + } + + #[test] + fn fifo_ttl_then_limit_additional_drops_blob_unit() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + // Create two tables; we will expire them via time override and force additional drops via limit. + tree.insert("a", "$", 0); + tree.flush_active_memtable(0)?; + tree.insert("b", "$", 1); + tree.flush_active_memtable(1)?; + + crate::time::set_unix_timestamp_for_test(Some(std::time::Duration::from_secs(10_000_000))); + + // TTL=1s will mark both expired; very small limit ensures size-based collection path is also exercised. + let fifo = Arc::new(Strategy::new(1, Some(1))); + tree.compact(fifo, 2)?; + + assert_eq!(0, tree.table_count()); + + crate::time::set_unix_timestamp_for_test(None); + Ok(()) + } +} diff --git a/crates/lsm-tree/src/compaction/filter.rs b/crates/lsm-tree/src/compaction/filter.rs new file mode 100644 index 000000000..e1f6f045b --- /dev/null +++ b/crates/lsm-tree/src/compaction/filter.rs @@ -0,0 +1,278 @@ +//! Definitions for compaction filters +//! +//! Compaction filters allow users to run custom logic during compactions, e.g. custom cleanup rules such as TTL. +//! Because compactions run in background workers, using compactions filters instead of scans can massively increase the efficiency of the storage engine. +use crate::{ + coding::{Decode, Encode}, + compaction::{ + stream::{StreamFilter, StreamFilterVerdict}, + worker::Options, + }, + key::InternalKey, + version::Version, + vlog::{Accessor, BlobFileWriter, ValueHandle}, + BlobIndirection, InternalValue, KvSeparationOptions, UserKey, UserValue, ValueType, +}; +use std::{panic::RefUnwindSafe, path::Path}; + +/// Verdict returned by a [`CompactionFilter`]. +#[non_exhaustive] +#[derive(Debug, Default)] +pub enum Verdict { + /// Keeps the item. + #[default] + Keep, + + /// Removes the item. + Remove, + + /// Removes the item and replace it with a weak tombstone. + /// + /// This may cause old versions of this item to be resurrected. + /// The semantics of this operation are identical to [`remove_weak`](crate::AbstractTree::remove_weak). + RemoveWeak, + + /// Replaces the value of the item. + ReplaceValue(UserValue), + + /// Destroys a value - does not leave behind a tombstone. + /// + /// Only use in situations where you absolutely 100% know your + /// item key is never written or updated multiple times. + Destroy, +} + +/// Trait to implement custom compaction filters +pub trait CompactionFilter: Send { + /// Returns whether an item should be kept during compaction. + /// + /// # Panicking + /// + /// This function should NOT panic. + /// + /// # Errors + /// + /// Returning an error will abort the running compaction. + /// This should only be done when **strictly** necessary, such as when fetching a value fails. + fn filter_item(&mut self, item: ItemAccessor<'_>, ctx: &Context) -> crate::Result; + + // TODO: how would we go about adding custom properties to tables? + // a function that gets called every time a table is cut...? e.g.: + // fn on_table_cut(&mut self, ctx: ...) {} + + /// Called when compaction is finished. + /// + /// # Panicking + /// + /// This function should NOT panic. + fn finish(self: Box) {} +} + +/// Context passed to compaction filters and factories for each compaction run +#[non_exhaustive] +#[derive(Debug)] +pub struct Context { + /// Whether we are compacting into the last level. + pub is_last_level: bool, +} + +/// Trait that creates compaction filter objects for each compaction +pub trait Factory: Send + Sync + RefUnwindSafe { + /// Returns the compaction filter name. + /// + /// This is currently only used for logging purposes. + fn name(&self) -> &str; + + /// Returns a new compaction filter. + /// + /// # Panicking + /// + /// This function should NOT panic. + fn make_filter(&self, ctx: &Context) -> Box; +} + +struct AccessorShared<'a> { + opts: &'a Options, + version: &'a Version, + blobs_folder: &'a Path, +} + +impl AccessorShared<'_> { + /// Fetches a value from the blob store. + fn get_indirect_value( + &self, + user_key: &[u8], + vhandle: &ValueHandle, + ) -> crate::Result> { + Accessor::new(&self.version.blob_files).get( + self.opts.tree_id, + self.blobs_folder, + user_key, + vhandle, + &self.opts.config.cache, + ) + } +} + +/// Accessor for the key/value from a compaction filter +pub struct ItemAccessor<'a> { + item: &'a InternalValue, + shared: &'a AccessorShared<'a>, +} + +impl<'a> ItemAccessor<'a> { + /// Get the key of this item. + #[must_use] + pub fn key(&self) -> &'a UserKey { + &self.item.key.user_key + } + + /// Returns whether this item's value is stored separately. + #[must_use] + #[doc(hidden)] + pub fn is_indirection(&self) -> bool { + self.item.key.value_type.is_indirection() + } + + /// Get the value of this item. + /// + /// # Errors + /// + /// This method will return an error if blob retrieval fails. + pub fn value(&self) -> crate::Result { + match self.item.key.value_type { + crate::ValueType::Value => Ok(self.item.value.clone()), + crate::ValueType::Indirection => { + // Resolve and read the value from a blob file. + let mut reader = &self.item.value[..]; + let indirection = BlobIndirection::decode_from(&mut reader)?; + let vhandle = indirection.vhandle; + + let value = self + .shared + .get_indirect_value(&self.item.key.user_key, &vhandle)?; + + if let Some(value) = value { + Ok(value) + } else { + log::error!( + "failed to read referenced blob file during execution of compaction filter. key: {:?}, vptr: {indirection:?}", + self.item.key, + ); + Err(crate::Error::Unrecoverable) + } + } + crate::ValueType::WeakTombstone | crate::ValueType::Tombstone => { + unreachable!("tombstones are filtered out before calling filter") + } + } + } +} + +/// Adapts a [`CompactionFilter`] to a [`StreamFilter`] +// +// NOTE: this slightly helps insulate CompactionStream from lifetime spam +pub(crate) struct StreamFilterAdapter<'a, 'b: 'a> { + filter: Option<&'a mut (dyn CompactionFilter + 'b)>, + shared: AccessorShared<'a>, + blob_opts: Option<&'a KvSeparationOptions>, + blob_writer: &'a mut Option, + ctx: &'a Context, +} + +impl<'a, 'b: 'a> StreamFilterAdapter<'a, 'b> { + pub fn new( + filter: Option<&'a mut (dyn CompactionFilter + 'b)>, + opts: &'a Options, + version: &'a Version, + blobs_folder: &'a Path, + blob_writer: &'a mut Option, + ctx: &'a Context, + ) -> Self { + Self { + filter, + shared: AccessorShared { + opts, + version, + blobs_folder, + }, + blob_opts: opts.config.kv_separation_opts.as_ref(), + blob_writer, + ctx, + } + } + + /// Redirects a write to a blob file if KV separation is enabled and + /// the value meets the separation threshold. + fn handle_write( + &mut self, + prev_key: &InternalKey, + new_value: UserValue, + ) -> crate::Result<(ValueType, UserValue)> { + let Some(blob_opts) = self.blob_opts else { + return Ok((ValueType::Value, new_value)); + }; + + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + let value_size = new_value.len() as u32; + + if value_size < blob_opts.separation_threshold { + return Ok((ValueType::Value, new_value)); + } + + let writer = if let Some(writer) = self.blob_writer { + writer + } else { + // Instantiate writer as necessary + let writer = BlobFileWriter::new( + self.shared.opts.blob_file_id_generator.clone(), + self.shared.blobs_folder, + self.shared.opts.tree_id, + self.shared.opts.config.descriptor_table.clone(), + )? + .use_target_size(blob_opts.file_target_size) + .use_compression(blob_opts.compression); + + self.blob_writer.insert(writer) + }; + + let vhandle = writer.write(&prev_key.user_key, prev_key.seqno, &new_value)?; + + let indirection = BlobIndirection { + vhandle, + size: value_size, + }; + + Ok((ValueType::Indirection, indirection.encode_into_vec().into())) + } +} + +impl<'a, 'b: 'a> StreamFilter for StreamFilterAdapter<'a, 'b> { + fn filter_item(&mut self, item: &InternalValue) -> crate::Result { + let Some(filter) = self.filter.as_mut() else { + return Ok(StreamFilterVerdict::Keep); + }; + + match filter.filter_item( + ItemAccessor { + item, + shared: &self.shared, + }, + self.ctx, + )? { + Verdict::Destroy => Ok(StreamFilterVerdict::Drop), + Verdict::Keep => Ok(StreamFilterVerdict::Keep), + Verdict::Remove => Ok(StreamFilterVerdict::Replace(( + ValueType::Tombstone, + UserValue::empty(), + ))), + Verdict::RemoveWeak => Ok(StreamFilterVerdict::Replace(( + ValueType::WeakTombstone, + UserValue::empty(), + ))), + Verdict::ReplaceValue(new_value) => self + .handle_write(&item.key, new_value) + .map(StreamFilterVerdict::Replace), + } + } +} diff --git a/crates/lsm-tree/src/compaction/flavour.rs b/crates/lsm-tree/src/compaction/flavour.rs new file mode 100644 index 000000000..9a5b34723 --- /dev/null +++ b/crates/lsm-tree/src/compaction/flavour.rs @@ -0,0 +1,543 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::blob_tree::handle::BlobIndirection; +use crate::blob_tree::FragmentationMap; +use crate::coding::{Decode, Encode}; +use crate::compaction::worker::Options; +use crate::compaction::Input as CompactionPayload; +use crate::file::TABLES_FOLDER; +use crate::table::multi_writer::MultiWriter; +use crate::version::{SuperVersions, Version}; +use crate::vlog::blob_file::scanner::ScanEntry; +use crate::vlog::{BlobFileId, BlobFileMergeScanner, BlobFileWriter}; +use crate::{BlobFile, HashSet, InternalValue, Table}; +use std::iter::Peekable; +use std::time::Instant; + +/// Drains all blobs that come "before" the given vptr. +fn drain_blobs>>( + scanner: &mut Peekable, + key: &[u8], + vptr: &BlobIndirection, +) -> crate::Result<()> { + loop { + let Some(blob) = scanner.next_if(|x| match x { + Ok((entry, blob_file_id)) => { + entry.key != key + || (*blob_file_id != vptr.vhandle.blob_file_id) + || (entry.offset < vptr.vhandle.offset) + } + Err(_) => true, + }) else { + break; + }; + let (entry, _) = blob?; + + assert!(entry.key <= key, "vptr was not matched with blob"); + } + + Ok(()) +} + +pub(super) fn prepare_table_writer( + version: &Version, + opts: &Options, + payload: &CompactionPayload, +) -> crate::Result { + let table_base_folder = opts.config.path.join(TABLES_FOLDER); + + let dst_lvl = payload.canonical_level.into(); + + let data_block_size = opts.config.data_block_size_policy.get(dst_lvl); + + let data_block_restart_interval = opts.config.data_block_restart_interval_policy.get(dst_lvl); + let index_block_restart_interval = opts.config.index_block_restart_interval_policy.get(dst_lvl); + + let data_block_compression = opts.config.data_block_compression_policy.get(dst_lvl); + let index_block_compression = opts.config.index_block_compression_policy.get(dst_lvl); + + let data_block_hash_ratio = opts.config.data_block_hash_ratio_policy.get(dst_lvl); + + let index_partitioning = opts.config.index_block_partitioning_policy.get(dst_lvl); + let filter_partitioning = opts.config.filter_block_partitioning_policy.get(dst_lvl); + + log::debug!( + "Compacting tables {:?} into L{} (canonical L{}), target_size={}, data_block_restart_interval={data_block_restart_interval}, index_block_restart_interval={index_block_restart_interval}, data_block_size={data_block_size}, data_block_compression={data_block_compression:?}, index_block_compression={index_block_compression:?}, mvcc_gc_watermark={}", + payload.table_ids, + payload.dest_level, + payload.canonical_level, + payload.target_size, + opts.mvcc_gc_watermark, + ); + + let mut table_writer = MultiWriter::new( + table_base_folder, + opts.table_id_generator.clone(), + payload.target_size, + payload.dest_level, + )?; + + if index_partitioning { + table_writer = table_writer.use_partitioned_index(); + } + if filter_partitioning { + table_writer = table_writer.use_partitioned_filter(); + } + + #[expect(clippy::cast_possible_truncation, reason = "max key size = u16")] + let last_level = (version.level_count() - 1) as u8; + let is_last_level = payload.dest_level == last_level; + + Ok(table_writer + .use_data_block_restart_interval(data_block_restart_interval) + .use_index_block_restart_interval(index_block_restart_interval) + .use_data_block_compression(data_block_compression) + .use_data_block_size(data_block_size) + .use_data_block_hash_ratio(data_block_hash_ratio) + .use_index_block_compression(index_block_compression) + .use_bloom_policy({ + use crate::config::FilterPolicyEntry::{Bloom, None}; + use crate::table::filter::BloomConstructionPolicy; + + if is_last_level && opts.config.expect_point_read_hits { + BloomConstructionPolicy::BitsPerKey(0.0) + } else { + match opts + .config + .filter_policy + .get(usize::from(payload.dest_level)) + { + Bloom(policy) => policy, + None => BloomConstructionPolicy::BitsPerKey(0.0), + } + } + })) +} + +// TODO: find a better name +pub(super) trait CompactionFlavour { + fn write(&mut self, item: InternalValue) -> crate::Result<()>; + + #[warn(clippy::too_many_arguments)] + fn finish( + self: Box, + super_version: &mut SuperVersions, + opts: &Options, + payload: &CompactionPayload, + dst_lvl: usize, + blob_frag_map: FragmentationMap, + extra_blob_files: Vec, + ) -> crate::Result<()>; +} + +/// Compaction worker that will relocate blobs that sit in blob files that are being rewritten +pub struct RelocatingCompaction { + inner: StandardCompaction, + blob_scanner: Peekable, + blob_writer: BlobFileWriter, + rewriting_blob_file_ids: HashSet, + rewriting_blob_files: Vec, +} + +impl RelocatingCompaction { + pub fn new( + inner: StandardCompaction, + blob_scanner: Peekable, + blob_writer: BlobFileWriter, + rewriting_blob_files: Vec, + ) -> Self { + Self { + inner, + blob_scanner, + blob_writer, + rewriting_blob_file_ids: rewriting_blob_files.iter().map(BlobFile::id).collect(), + rewriting_blob_files, + } + } + + // TODO: vvv validate/unit test this vvv + fn drain_blobs(&mut self, key: &[u8], indirection: &BlobIndirection) -> crate::Result<()> { + drain_blobs(&mut self.blob_scanner, key, indirection) + } +} + +impl CompactionFlavour for RelocatingCompaction { + fn write(&mut self, item: InternalValue) -> crate::Result<()> { + if item.key.value_type.is_indirection() { + let mut reader = &item.value[..]; + + let indirection = BlobIndirection::decode_from(&mut reader).inspect_err(|e| { + log::error!("Failed to deserialize blob indirection {item:?}: {e:?}"); + })?; + + log::trace!( + "{:?}:{} => encountered indirection: {indirection:?}", + item.key.user_key, + item.key.seqno, + ); + + let indirection = if self + .rewriting_blob_file_ids + .contains(&indirection.vhandle.blob_file_id) + { + self.drain_blobs(&item.key.user_key, &indirection)?; + + #[expect(clippy::expect_used, reason = "vptr is expected to match with blob")] + let (blob_entry, blob_file_id) = self + .blob_scanner + .next() + .expect("vptr was not matched with blob (scanner is unexpectedly exhausted)")?; + + assert_eq!( + blob_file_id, indirection.vhandle.blob_file_id, + "matched blob has different blob file ID than vptr", + ); + assert_eq!( + blob_entry.key, item.key.user_key, + "matched blob has different key than vptr", + ); + assert_eq!( + blob_entry.offset, indirection.vhandle.offset, + "matched blob has different offset than vptr", + ); + + log::trace!( + "=> use blob: {:?}:{} offset: {} from BF {}", + blob_entry.key, + blob_entry.seqno, + blob_entry.offset, + blob_file_id, + ); + + log::trace!("RELOCATE to {indirection:?}"); + + let new_indirection = BlobIndirection { + vhandle: self.blob_writer.write_raw( + &item.key.user_key, + item.key.seqno, + &blob_entry.value, + blob_entry.uncompressed_len, + )?, + size: indirection.size, + }; + + debug_assert_eq!( + new_indirection.vhandle.on_disk_size, indirection.vhandle.on_disk_size, + "redirecting blob should not change its size", + ); + + self.inner + .table_writer + .write(InternalValue::from_components( + item.key.user_key, + new_indirection.encode_into_vec(), + item.key.seqno, + crate::ValueType::Indirection, + ))?; + + new_indirection + } else { + // This blob is not part of the rewritten blob files + // So just pass it through + log::trace!("Pass through {indirection:?} because it is not being relocated"); + self.inner.table_writer.write(item)?; + + indirection + }; + + self.inner.table_writer.register_blob(indirection); + } else { + self.inner.table_writer.write(item)?; + } + + Ok(()) + } + + fn finish( + mut self: Box, + super_version: &mut SuperVersions, + opts: &Options, + payload: &CompactionPayload, + dst_lvl: usize, + blob_frag_map_diff: FragmentationMap, + extra_blob_files: Vec, + ) -> crate::Result<()> { + log::debug!( + "Relocating compaction done in {:?}", + self.inner.start.elapsed(), + ); + + let table_ids_to_delete = std::mem::take(&mut self.inner.tables_to_rewrite); + + let created_tables = self.inner.consume_writer(opts, dst_lvl)?; + let mut created_blob_files = self.blob_writer.finish()?; + created_blob_files.extend(extra_blob_files); + + let mut blob_files_to_drop = self.rewriting_blob_files; + + let current_version = super_version.latest_version(); + + for blob_file in current_version.version.blob_files.iter() { + if blob_file.is_dead(current_version.version.gc_stats()) { + blob_files_to_drop.push(blob_file.clone()); + } + } + + super_version.upgrade_version( + &opts.config.path, + |current| { + let mut copy = current.clone(); + + copy.version = copy.version.with_merge( + &payload.table_ids.iter().copied().collect::>(), + &created_tables, + payload.dest_level as usize, + if blob_frag_map_diff.is_empty() { + None + } else { + Some(blob_frag_map_diff) + }, + created_blob_files, + &blob_files_to_drop + .iter() + .map(BlobFile::id) + .collect::>(), + ); + + Ok(copy) + }, + &opts.global_seqno, + &opts.visible_seqno, + )?; + + // NOTE: If the application were to crash >here< it's fine + // The tables/blob files are not referenced anymore, and will be + // cleaned up upon recovery + for table in table_ids_to_delete { + table.mark_as_deleted(); + } + + for blob_file in blob_files_to_drop { + blob_file.mark_as_deleted(); + } + + Ok(()) + } +} + +/// Standard compaction worker that just passes through all its data +pub struct StandardCompaction { + start: Instant, + table_writer: MultiWriter, + tables_to_rewrite: Vec
, +} + +impl StandardCompaction { + pub fn new(table_writer: MultiWriter, tables_to_rewrite: Vec
) -> Self { + Self { + start: Instant::now(), + table_writer, + tables_to_rewrite, + } + } + + fn consume_writer(self, opts: &Options, dst_lvl: usize) -> crate::Result> { + let table_base_folder = self.table_writer.base_path.clone(); + + let pin_filter = opts.config.filter_block_pinning_policy.get(dst_lvl); + let pin_index = opts.config.index_block_pinning_policy.get(dst_lvl); + + self.table_writer + .finish()? + .into_iter() + .map(|(table_id, checksum)| -> crate::Result
{ + Table::recover( + table_base_folder.join(table_id.to_string()), + checksum, + 0, + opts.tree_id, + opts.config.cache.clone(), + opts.config.descriptor_table.clone(), + pin_filter, + pin_index, + #[cfg(feature = "metrics")] + opts.metrics.clone(), + ) + }) + .collect::>>() + } +} + +impl CompactionFlavour for StandardCompaction { + fn write(&mut self, item: InternalValue) -> crate::Result<()> { + let indirection = if item.key.value_type.is_indirection() { + Some({ + let mut reader = &item.value[..]; + BlobIndirection::decode_from(&mut reader)? + }) + } else { + None + }; + + self.table_writer.write(item)?; + + if let Some(indirection) = indirection { + self.table_writer.register_blob(indirection); + } + + Ok(()) + } + + fn finish( + mut self: Box, + super_version: &mut SuperVersions, + opts: &Options, + payload: &CompactionPayload, + dst_lvl: usize, + blob_frag_map: FragmentationMap, + extra_blob_files: Vec, + ) -> crate::Result<()> { + log::debug!("Compaction done in {:?}", self.start.elapsed()); + + let table_ids_to_delete = std::mem::take(&mut self.tables_to_rewrite); + + let created_tables = self.consume_writer(opts, dst_lvl)?; + + let mut blob_files_to_drop = Vec::default(); + + let current_version = super_version.latest_version(); + + for blob_file in current_version.version.blob_files.iter() { + if blob_file.is_dead(current_version.version.gc_stats()) { + blob_files_to_drop.push(blob_file.clone()); + } + } + + super_version.upgrade_version( + &opts.config.path, + |current| { + let mut copy = current.clone(); + + copy.version = copy.version.with_merge( + &payload.table_ids.iter().copied().collect::>(), + &created_tables, + payload.dest_level as usize, + if blob_frag_map.is_empty() { + None + } else { + Some(blob_frag_map) + }, + extra_blob_files, + &blob_files_to_drop + .iter() + .map(BlobFile::id) + .collect::>(), + ); + + Ok(copy) + }, + &opts.global_seqno, + &opts.visible_seqno, + )?; + + // NOTE: If the application were to crash >here< it's fine + // The tables are not referenced anymore, and will be + // cleaned up upon recovery + for table in table_ids_to_delete { + table.mark_as_deleted(); + } + + for blob_file in blob_files_to_drop { + blob_file.mark_as_deleted(); + } + + Ok(()) + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::{vlog::ValueHandle, UserKey, UserValue}; + + #[expect(clippy::unnecessary_wraps)] + fn entry( + blob_file_id: BlobFileId, + key: &[u8], + offset: u64, + ) -> crate::Result<(ScanEntry, BlobFileId)> { + Ok(( + ScanEntry { + key: UserKey::from(key), + offset, + seqno: 0, + uncompressed_len: 0, + value: UserValue::empty(), + }, + blob_file_id, + )) + } + + #[test] + fn drain_blobs_simple() -> crate::Result<()> { + let mut iter = [ + entry(0, b"a", 0), + entry(0, b"a", 1), + entry(0, b"a", 2), + entry(0, b"a", 3), + entry(0, b"a", 4), + ] + .into_iter() + .peekable(); + + drain_blobs( + &mut iter, + b"a", + &BlobIndirection { + size: 0, + vhandle: ValueHandle { + blob_file_id: 0, + offset: 4, + on_disk_size: 0, + }, + }, + )?; + + assert_eq!(entry(0, b"a", 4)?, iter.next().unwrap()?); + + Ok(()) + } + + #[test] + fn drain_blobs_multiple_keys() -> crate::Result<()> { + let mut iter = [ + entry(0, b"a", 0), + entry(0, b"b", 0), + entry(0, b"c", 0), + entry(0, b"d", 0), + entry(0, b"e", 0), + ] + .into_iter() + .peekable(); + + drain_blobs( + &mut iter, + b"e", + &BlobIndirection { + size: 0, + vhandle: ValueHandle { + blob_file_id: 0, + offset: 0, + on_disk_size: 0, + }, + }, + )?; + + assert_eq!(entry(0, b"e", 0)?, iter.next().unwrap()?); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/compaction/leveled/mod.rs b/crates/lsm-tree/src/compaction/leveled/mod.rs new file mode 100644 index 000000000..f4bd190ab --- /dev/null +++ b/crates/lsm-tree/src/compaction/leveled/mod.rs @@ -0,0 +1,579 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +#[cfg(test)] +mod test; + +use super::{Choice, CompactionStrategy, Input as CompactionInput}; +use crate::{ + compaction::state::{hidden_set::HiddenSet, CompactionState}, + config::Config, + slice_windows::{GrowingWindowsExt, ShrinkingWindowsExt}, + table::{util::aggregate_run_key_range, Table}, + version::{Run, Version}, + HashSet, TableId, +}; + +/// Tries to find the most optimal compaction set from one level into the other. +fn pick_minimal_compaction( + curr_run: &Run
, + next_run: Option<&Run
>, + hidden_set: &HiddenSet, + _overshoot: u64, + table_base_size: u64, +) -> Option<(HashSet, bool)> { + // NOTE: Find largest trivial move (if it exists) + if let Some(window) = curr_run.shrinking_windows().find(|window| { + if hidden_set.is_blocked(window.iter().map(Table::id)) { + // IMPORTANT: Compaction is blocked because of other + // on-going compaction + return false; + } + + let Some(next_run) = &next_run else { + // No run in next level, so we can trivially move + return true; + }; + + let key_range = aggregate_run_key_range(window); + + next_run.get_overlapping(&key_range).is_empty() + }) { + let ids = window.iter().map(Table::id).collect(); + return Some((ids, true)); + } + + // NOTE: Look for merges + if let Some(next_run) = &next_run { + next_run + .growing_windows() + .take_while(|window| { + // Cap at 50x tables per compaction for now + // + // At this point, all compactions are too large anyway + // so we can escape early + let next_level_size = window.iter().map(Table::file_size).sum::(); + next_level_size <= (50 * table_base_size) + }) + .filter_map(|window| { + if hidden_set.is_blocked(window.iter().map(Table::id)) { + // IMPORTANT: Compaction is blocked because of other + // on-going compaction + return None; + } + + let key_range = aggregate_run_key_range(window); + + // Pull in all contained tables in current level into compaction + let curr_level_pull_in = curr_run.get_contained(&key_range); + + let curr_level_size = curr_level_pull_in.iter().map(Table::file_size).sum::(); + + if curr_level_size == 0 { + return None; + } + + // TODO: toggling this statement can deadlock compactions because if there are only larger-than-overshoot + // compactions, they would not be chosen + // if curr_level_size < overshoot { + // return None; + // } + + if hidden_set.is_blocked(curr_level_pull_in.iter().map(Table::id)) { + // IMPORTANT: Compaction is blocked because of other + // on-going compaction + return None; + } + + let next_level_size = window.iter().map(Table::file_size).sum::(); + + let compaction_bytes = curr_level_size + next_level_size; + + #[expect(clippy::cast_precision_loss)] + let write_amp = (next_level_size as f32) / (curr_level_size as f32); + + Some((window, curr_level_pull_in, write_amp, compaction_bytes)) + }) + // Find the compaction with the smallest write set + .min_by_key(|(_, _, _waf, bytes)| *bytes) + .map(|(window, curr_level_pull_in, _, _)| { + let mut ids: HashSet<_> = window.iter().map(Table::id).collect(); + ids.extend(curr_level_pull_in.iter().map(Table::id)); + (ids, false) + }) + } else { + None + } +} + +#[doc(hidden)] +pub const NAME: &str = "LeveledCompaction"; + +/// Leveled compaction strategy (LCS) +/// +/// When a level reaches some threshold size, parts of it are merged into overlapping tables in the next level. +/// +/// Each level Ln for n >= 2 can have up to `level_base_size * ratio^(n - 1)` tables. +/// +/// LCS suffers from comparatively high write amplification, but has decent read amplification and great space amplification (~1.1x). +/// +/// LCS is the recommended compaction strategy to use. +/// +/// More info here: +#[derive(Clone)] +pub struct Strategy { + l0_threshold: u8, + + /// The target table size as disk (possibly compressed). + target_size: u64, + + /// Size ratio between levels of the LSM tree (a.k.a fanout, growth rate) + level_ratio_policy: Vec, +} + +impl Default for Strategy { + fn default() -> Self { + Self { + l0_threshold: 4, + target_size:/* 64 MiB */ 64 * 1_024 * 1_024, + level_ratio_policy: vec![10.0], + } + } +} + +impl Strategy { + /// Sets the growth ratio between levels. + /// + /// Same as `set_max_bytes_for_level_multiplier` in `RocksDB`. + /// + /// Default = [10.0] + #[must_use] + pub fn with_level_ratio_policy(mut self, policy: Vec) -> Self { + self.level_ratio_policy = policy; + self + } + + /// Sets the L0 threshold. + /// + /// When the number of tables in L0 reaches this threshold, + /// they are merged into L1. + /// + /// Same as `level0_file_num_compaction_trigger` in `RocksDB`. + /// + /// Default = 4 + #[must_use] + pub fn with_l0_threshold(mut self, threshold: u8) -> Self { + self.l0_threshold = threshold; + self + } + + /// Sets the table target size on disk (possibly compressed). + /// + /// Same as `target_file_size_base` in `RocksDB`. + /// + /// Default = 64 MiB + #[must_use] + pub fn with_table_target_size(mut self, bytes: u64) -> Self { + self.target_size = bytes; + self + } + + /// Calculates the size of L1. + fn level_base_size(&self) -> u64 { + self.target_size * u64::from(self.l0_threshold) + } + + /// Calculates the level target size. + /// + /// L1 = `level_base_size` + /// + /// L2 = `level_base_size * ratio` + /// + /// L3 = `level_base_size * ratio * ratio` + /// + /// ... + fn level_target_size(&self, canonical_level_idx: u8) -> u64 { + assert!( + canonical_level_idx >= 1, + "level_target_size does not apply to L0", + ); + + if canonical_level_idx == 1 { + // u64::from(self.target_size) + self.level_base_size() + } else { + #[expect( + clippy::cast_precision_loss, + reason = "precision loss is acceptable for level size calculations" + )] + let mut size = self.level_base_size() as f32; + + // NOTE: Minus 2 because |{L0, L1}| + for idx in 0..=(canonical_level_idx - 2) { + let ratio = self + .level_ratio_policy + .get(usize::from(idx)) + .copied() + .unwrap_or_else(|| self.level_ratio_policy.last().copied().unwrap_or(10.0)); + + size *= ratio; + } + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "size is always positive and will never even come close to u64::MAX" + )] + { + size as u64 + } + } + } +} + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + NAME + } + + fn get_config(&self) -> Vec { + vec![ + ( + crate::UserKey::from("leveled_l0_threshold"), + crate::UserValue::from(self.l0_threshold.to_le_bytes()), + ), + ( + crate::UserKey::from("leveled_target_size"), + crate::UserValue::from(self.target_size.to_le_bytes()), + ), + ( + crate::UserKey::from("leveled_level_ratio_policy"), + crate::UserValue::from({ + use byteorder::{LittleEndian, WriteBytesExt}; + + let mut v = vec![]; + + #[expect( + clippy::expect_used, + clippy::cast_possible_truncation, + reason = "writing into Vec should not fail; policies have length of 255 max" + )] + v.write_u8(self.level_ratio_policy.len() as u8) + .expect("cannot fail"); + + for &f in &self.level_ratio_policy { + #[expect(clippy::expect_used, reason = "writing into Vec should not fail")] + v.write_f32::(f).expect("cannot fail"); + } + + v + }), + ), + ] + } + + #[expect(clippy::too_many_lines)] + fn choose(&self, version: &Version, _: &Config, state: &CompactionState) -> Choice { + assert!(version.level_count() == 7, "should have exactly 7 levels"); + + // Trivial move into Lmax + 'trivial_lmax: { + let l0 = version.level(0).expect("first level should exist"); + + if !l0.is_empty() && l0.is_disjoint() { + let lmax_index = version.level_count() - 1; + + if (1..lmax_index) + .any(|idx| !version.level(idx).expect("level should exist").is_empty()) + { + // There are intermediary levels with data, cannot trivially move to Lmax + break 'trivial_lmax; + } + + let lmax = version.level(lmax_index).expect("last level should exist"); + + if !lmax + .aggregate_key_range() + .overlaps_with_key_range(&l0.aggregate_key_range()) + { + return Choice::Move(CompactionInput { + table_ids: l0.list_ids(), + dest_level: lmax_index as u8, + canonical_level: 1, + target_size: self.target_size, + }); + } + } + } + + // Find the level that corresponds to L1 + #[expect(clippy::map_unwrap_or)] + let first_non_empty_level = version + .iter_levels() + .enumerate() + .skip(1) + .find(|(_, lvl)| !lvl.is_empty()) + .map(|(idx, _)| idx) + .unwrap_or_else(|| version.level_count() - 1); + + let mut canonical_l1_idx = first_non_empty_level; + + // Number of levels we have to shift to get from the actual level idx to the canonical + let mut level_shift = canonical_l1_idx - 1; + + if canonical_l1_idx > 1 && version.iter_levels().skip(1).any(|lvl| !lvl.is_empty()) { + let need_new_l1 = version + .iter_levels() + .enumerate() + .skip(1) + .filter(|(_, lvl)| !lvl.is_empty()) + .all(|(idx, level)| { + let level_size = level + .iter() + .flat_map(|x| x.iter()) + // NOTE: Take bytes that are already being compacted into account, + // otherwise we may be overcompensating + .filter(|x| !state.hidden_set().is_hidden(x.id())) + .map(Table::file_size) + .sum::(); + + #[expect( + clippy::cast_possible_truncation, + reason = "level index is bounded by level count (7, technically 255)" + )] + let target_size = self.level_target_size((idx - level_shift) as u8); + + level_size > target_size + }); + + // Move up L1 one level if all current levels are at capacity + if need_new_l1 { + canonical_l1_idx -= 1; + level_shift -= 1; + } + } + + // Trivial move into L1 + 'trivial: { + let first_level = version.l0(); + let target_level_idx = first_non_empty_level.min(canonical_l1_idx); + + if first_level.run_count() == 1 { + if version.level_is_busy(0, state.hidden_set()) + || version.level_is_busy(target_level_idx, state.hidden_set()) + { + break 'trivial; + } + + let Some(target_level) = &version.level(target_level_idx) else { + break 'trivial; + }; + + if target_level.run_count() != 1 { + break 'trivial; + } + + let key_range = first_level.aggregate_key_range(); + + // Get overlapping tables in next level + let get_overlapping = target_level + .iter() + .flat_map(|run| run.get_overlapping(&key_range)) + .map(Table::id) + .next(); + + if get_overlapping.is_none() && first_level.is_disjoint() { + #[expect( + clippy::cast_possible_truncation, + reason = "level index is bounded by level count (7)" + )] + return Choice::Move(CompactionInput { + table_ids: first_level.list_ids(), + dest_level: target_level_idx as u8, + canonical_level: 1, + target_size: self.target_size, + }); + } + } + } + + // Scoring + let mut scores = [(/* score */ 0.0, /* overshoot */ 0u64); 7]; + + { + // TODO(weak-tombstone-rewrite): incorporate `Table::weak_tombstone_count` and + // `Table::weak_tombstone_reclaimable` when computing level scores so rewrite + // decisions can prioritize tables that would free the most reclaimable values. + + // Score first level + let first_level = version.l0(); + + if first_level.table_count() >= usize::from(self.l0_threshold) { + #[expect( + clippy::cast_precision_loss, + reason = "precision loss is acceptable for scoring calculations" + )] + let ratio = (first_level.table_count() as f64) / f64::from(self.l0_threshold); + scores[0] = (ratio, 0); + } + + // Score L1+ + for (idx, level) in version.iter_levels().enumerate().skip(1) { + if level.is_empty() { + continue; + } + + let level_size = level + .iter() + .flat_map(|x| x.iter()) + // NOTE: Take bytes that are already being compacted into account, + // otherwise we may be overcompensating + .filter(|x| !state.hidden_set().is_hidden(x.id())) + .map(Table::file_size) + .sum::(); + + #[expect( + clippy::cast_possible_truncation, + reason = "level index is bounded by level count (7, technically 255)" + )] + let target_size = self.level_target_size((idx - level_shift) as u8); + + // NOTE: We check for level length above + #[expect(clippy::indexing_slicing)] + if level_size > target_size { + #[expect( + clippy::cast_precision_loss, + reason = "precision loss is acceptable for scoring calculations" + )] + let score = level_size as f64 / target_size as f64; + scores[idx] = (score, level_size - target_size); + + // NOTE: Force a trivial move + if version + .level(idx + 1) + .is_some_and(|next_level| next_level.is_empty()) + { + scores[idx] = (99.99, 999); + } + } + } + + // NOTE: Never score Lmax + { + scores[6] = (0.0, 0); + } + } + + // Choose compaction + #[expect(clippy::expect_used, reason = "highest score is expected to exist")] + let (level_idx_with_highest_score, (score, overshoot_bytes)) = scores + .into_iter() + .enumerate() + .max_by(|(_, (score_a, _)), (_, (score_b, _))| { + score_a + .partial_cmp(score_b) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .expect("should have highest score somewhere"); + + if score < 1.0 { + return Choice::DoNothing; + } + + // We choose L0->L1 compaction + if level_idx_with_highest_score == 0 { + let Some(first_level) = version.level(0) else { + return Choice::DoNothing; + }; + + if version.level_is_busy(0, state.hidden_set()) + || version.level_is_busy(canonical_l1_idx, state.hidden_set()) + { + return Choice::DoNothing; + } + + let Some(target_level) = &version.level(canonical_l1_idx) else { + return Choice::DoNothing; + }; + + let mut table_ids = first_level.list_ids(); + + let key_range = first_level.aggregate_key_range(); + + // Get overlapping tables in next level + let target_level_overlapping_table_ids: Vec<_> = target_level + .iter() + .flat_map(|run| run.get_overlapping(&key_range)) + .map(Table::id) + .collect(); + + table_ids.extend(&target_level_overlapping_table_ids); + + #[expect( + clippy::cast_possible_truncation, + reason = "level index is bounded by level count (7, technically 255)" + )] + let choice = CompactionInput { + table_ids, + dest_level: canonical_l1_idx as u8, + canonical_level: 1, + target_size: self.target_size, + }; + + if target_level_overlapping_table_ids.is_empty() && first_level.is_disjoint() { + return Choice::Move(choice); + } + return Choice::Merge(choice); + } + + // We choose L1+ compaction instead + + // NOTE: Level count is 255 max + #[expect(clippy::cast_possible_truncation)] + let curr_level_index = level_idx_with_highest_score as u8; + + let next_level_index = curr_level_index + 1; + + let Some(level) = version.level(level_idx_with_highest_score) else { + return Choice::DoNothing; + }; + + let Some(next_level) = version.level(next_level_index as usize) else { + return Choice::DoNothing; + }; + + debug_assert!(level.is_disjoint(), "level should be disjoint"); + debug_assert!(next_level.is_disjoint(), "next level should be disjoint"); + + #[expect( + clippy::expect_used, + reason = "first run should exist because score is >0.0" + )] + let Some((table_ids, can_trivial_move)) = pick_minimal_compaction( + level.first_run().expect("should have exactly one run"), + next_level.first_run().map(std::ops::Deref::deref), + state.hidden_set(), + overshoot_bytes, + self.target_size, + ) else { + return Choice::DoNothing; + }; + + #[expect( + clippy::cast_possible_truncation, + reason = "level shift is bounded by level count (7, technically 255)" + )] + let choice = CompactionInput { + table_ids, + dest_level: next_level_index, + canonical_level: next_level_index - (level_shift as u8), + target_size: self.target_size, + }; + + if can_trivial_move && level.is_disjoint() { + return Choice::Move(choice); + } + Choice::Merge(choice) + } +} diff --git a/crates/lsm-tree/src/compaction/leveled/test.rs b/crates/lsm-tree/src/compaction/leveled/test.rs new file mode 100644 index 000000000..5486c86d8 --- /dev/null +++ b/crates/lsm-tree/src/compaction/leveled/test.rs @@ -0,0 +1,177 @@ +use super::*; +use crate::{AbstractTree, Config, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn leveled_empty_levels() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let strategy = Arc::new(Strategy::default()); + tree.compact(strategy, 0)?; + + assert_eq!(0, tree.table_count()); + Ok(()) +} + +#[test] +fn leveled_l0_below_limit() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..3u8 { + tree.insert([b'k', i].as_slice(), "v", 0); + tree.flush_active_memtable(0)?; + } + + let before = tree.table_count(); + assert_eq!(3, before); + + let strategy = Arc::new(Strategy::default()); + tree.compact(strategy, 0)?; + + assert_eq!(before, tree.table_count()); + + Ok(()) +} + +#[test] +fn leveled_l0_reached_limit() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..4u8 { + // NOTE: Tables need to overlap + tree.insert("a", "v", 0); + tree.insert([b'k', i].as_slice(), "v", 0); + tree.insert("z", "v", 0); + tree.flush_active_memtable(0)?; + } + + assert_eq!(4, tree.table_count()); + + let strategy = Arc::new(Strategy::default()); + tree.compact(strategy, 0)?; + + assert_eq!(1, tree.table_count()); + + Ok(()) +} + +#[test] +fn leveled_l0_reached_limit_disjoint() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..4u8 { + tree.insert([b'k', i].as_slice(), "v", 0); + tree.flush_active_memtable(0)?; + } + + assert_eq!(4, tree.table_count()); + + let strategy = Arc::new(Strategy::default()); + tree.compact(strategy, 0)?; + + assert_eq!(4, tree.table_count()); + + Ok(()) +} + +#[test] +fn leveled_l0_reached_limit_disjoint_l1() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..4 { + // NOTE: Tables need to overlap + tree.insert("a", "v", i); + tree.insert("b", "v", i); + tree.flush_active_memtable(0)?; + } + + let fifo = Arc::new(Strategy::default()); + tree.compact(fifo, 0)?; + + assert_eq!(1, tree.table_count()); + + for i in 0..4u8 { + tree.insert([b'k', i].as_slice(), "v", 0); + tree.flush_active_memtable(0)?; + } + + assert_eq!(5, tree.table_count()); + + let strategy = Arc::new(Strategy::default()); + tree.compact(strategy, 0)?; + + assert_eq!(5, tree.table_count()); + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn leveled_sequential_inserts() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let strategy = Arc::new(Strategy { + target_size: 1, + ..Default::default() + }); + + let mut table_count = 0; + + for k in 0u64..100 { + table_count += 1; + + tree.insert(k.to_be_bytes(), "", 0); + tree.flush_active_memtable(0)?; + + assert_eq!(table_count, tree.table_count()); + tree.compact(strategy.clone(), 0)?; + assert_eq!(table_count, tree.table_count()); + + for idx in 1..=5 { + assert_eq!( + 0, + tree.current_version().level(idx).unwrap().len(), + "no tables should be in intermediary level (L{idx})", + ); + } + } + + Ok(()) +} diff --git a/crates/lsm-tree/src/compaction/maintenance.rs b/crates/lsm-tree/src/compaction/maintenance.rs new file mode 100644 index 000000000..aa0974c7a --- /dev/null +++ b/crates/lsm-tree/src/compaction/maintenance.rs @@ -0,0 +1,179 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy}; +use crate::{config::Config, level_manifest::LevelManifest, segment::Segment, HashSet, SegmentId}; + +const L0_SEGMENT_CAP: usize = 20; + +/// Maintenance compactor +/// +/// This is a hidden compaction strategy that may be called by other strategies. +/// +/// It cleans up L0 if it grows too large. +#[derive(Default)] +pub struct Strategy; + +/// Choose a run of segments that has the least file size sum. +/// +/// This minimizes the compaction time (+ write amp) for a set of segments we +/// want to partially compact. +pub fn choose_least_effort_compaction(segments: &[Segment], n: usize) -> HashSet { + let num_segments = segments.len(); + + // Ensure that n is not greater than the number of segments + assert!( + n <= num_segments, + "N must be less than or equal to the number of segments" + ); + + let windows = segments.windows(n); + + let window = windows + .min_by_key(|window| window.iter().map(|s| s.file_size()).sum::()) + .expect("should have at least one window"); + + window.iter().map(Segment::id).collect() +} + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + "MaintenanceStrategy" + } + + fn choose(&self, levels: &LevelManifest, _: &Config) -> Choice { + todo!() + } +} +/* +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + cache::Cache, + compaction::{Choice, CompactionStrategy}, + config::Config, + descriptor_table::FileDescriptorTable, + file::LEVELS_MANIFEST_FILE, + level_manifest::LevelManifest, + segment::{ + block::offset::BlockOffset, + block_index::{two_level_index::TwoLevelBlockIndex, BlockIndexImpl}, + file_offsets::FileOffsets, + meta::Metadata, + SegmentInner, + }, + super_segment::Segment, + KeyRange, + }; + use std::sync::{atomic::AtomicBool, Arc}; + use test_log::test; + + #[allow(clippy::expect_used)] + fn fixture_segment(id: SegmentId, created_at: u128) -> Segment { + todo!() + /* let cache = Arc::new(Cache::with_capacity_bytes(10 * 1_024 * 1_024)); + + let block_index = TwoLevelBlockIndex::new((0, id).into(), cache.clone()); + let block_index = Arc::new(BlockIndexImpl::TwoLevel(block_index)); + + SegmentInner { + tree_id: 0, + descriptor_table: Arc::new(FileDescriptorTable::new(512, 1)), + block_index, + + offsets: FileOffsets { + bloom_ptr: BlockOffset(0), + range_filter_ptr: BlockOffset(0), + index_block_ptr: BlockOffset(0), + metadata_ptr: BlockOffset(0), + range_tombstones_ptr: BlockOffset(0), + tli_ptr: BlockOffset(0), + pfx_ptr: BlockOffset(0), + }, + + metadata: Metadata { + data_block_count: 0, + index_block_count: 0, + data_block_size: 4_096, + index_block_size: 4_096, + created_at, + id, + file_size: 1, + compression: crate::segment::meta::CompressionType::None, + table_type: crate::segment::meta::TableType::Block, + item_count: 0, + key_count: 0, + key_range: KeyRange::new((vec![].into(), vec![].into())), + tombstone_count: 0, + range_tombstone_count: 0, + uncompressed_size: 0, + seqnos: (0, created_at as u64), + }, + cache, + + bloom_filter: Some(crate::bloom::BloomFilter::with_fp_rate(1, 0.1)), + + path: "a".into(), + is_deleted: AtomicBool::default(), + } + .into() */ + } + + #[test] + fn maintenance_empty_level() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + let compactor = Strategy; + + let levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + + assert_eq!( + compactor.choose(&levels, &Config::default()), + Choice::DoNothing + ); + + Ok(()) + } + + #[test] + fn maintenance_below_limit() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + let compactor = Strategy; + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + for id in 0..5 { + levels.add(fixture_segment(id, u128::from(id))); + } + + assert_eq!( + compactor.choose(&levels, &Config::default()), + Choice::DoNothing + ); + + Ok(()) + } + + #[test] + fn maintenance_l0_too_large() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + let compactor = Strategy; + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + for id in 0..(L0_SEGMENT_CAP + 2) { + levels.add(fixture_segment(id as u64, id as u128)); + } + + assert_eq!( + compactor.choose(&levels, &Config::default()), + Choice::Merge(crate::compaction::Input { + dest_level: 0, + segment_ids: [0, 1, 2].into_iter().collect::>(), + target_size: u64::MAX + }) + ); + + Ok(()) + } +} + */ diff --git a/crates/lsm-tree/src/compaction/major.rs b/crates/lsm-tree/src/compaction/major.rs new file mode 100644 index 000000000..0e973cc87 --- /dev/null +++ b/crates/lsm-tree/src/compaction/major.rs @@ -0,0 +1,57 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy, Input as CompactionInput}; +use crate::{ + compaction::state::CompactionState, config::Config, table::Table, version::Version, HashSet, +}; + +/// Compacts all tables into the last level +pub struct Strategy { + target_size: u64, +} + +impl Strategy { + /// Configures a new `Major` compaction strategy. + #[must_use] + pub fn new(target_size: u64) -> Self { + Self { target_size } + } +} + +impl Default for Strategy { + fn default() -> Self { + Self { + target_size: u64::MAX, + } + } +} + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + "MajorCompaction" + } + + fn choose(&self, version: &Version, cfg: &Config, state: &CompactionState) -> Choice { + let table_ids: HashSet<_> = version.iter_tables().map(Table::id).collect(); + + // NOTE: This should generally not occur because of the + // tree-level major compaction lock + // But just as a fail-safe... + let some_hidden = table_ids.iter().any(|&id| state.hidden_set().is_hidden(id)); + + if some_hidden { + Choice::DoNothing + } else { + let last_level_idx = cfg.level_count - 1; + + Choice::Merge(CompactionInput { + table_ids, + dest_level: last_level_idx, + canonical_level: last_level_idx, + target_size: self.target_size, + }) + } + } +} diff --git a/crates/lsm-tree/src/compaction/mod.rs b/crates/lsm-tree/src/compaction/mod.rs new file mode 100644 index 000000000..8aeea631c --- /dev/null +++ b/crates/lsm-tree/src/compaction/mod.rs @@ -0,0 +1,98 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +//! Contains compaction strategies + +pub(crate) mod fifo; +pub(crate) mod leveled; +// pub(crate) mod maintenance; +pub(crate) mod drop_range; +pub mod filter; +mod flavour; +pub(crate) mod major; +pub(crate) mod movedown; +pub(crate) mod pulldown; +pub(crate) mod state; +pub(crate) mod stream; +// pub(crate) mod tiered; +pub(crate) mod worker; + +pub use fifo::Strategy as Fifo; +pub use filter::{CompactionFilter, Factory, ItemAccessor, Verdict}; +pub use leveled::Strategy as Leveled; +// pub use tiered::Strategy as SizeTiered; + +pub use {fifo::NAME as FIFO_COMPACTION_NAME, leveled::NAME as LEVELED_COMPACTION_NAME}; + +/// Alias for `Leveled` +pub type Levelled = Leveled; + +#[doc(hidden)] +pub use movedown::Strategy as MoveDown; + +#[doc(hidden)] +pub use pulldown::Strategy as PullDown; + +use crate::{ + compaction::state::CompactionState, config::Config, version::Version, HashSet, KvPair, TableId, +}; + +/// Input for compactor +/// +/// The compaction strategy chooses which tables to compact and how. +/// That information is given to the compactor. +#[derive(Debug, Eq, PartialEq)] +pub struct Input { + /// Tables to compact + pub table_ids: HashSet, + + /// Level to put the created tables into + pub dest_level: u8, + + /// The logical level the tables are part of + pub canonical_level: u8, + + /// Table target size + /// + /// If a table merge reaches the size threshold, a new table is started. + /// This results in a sorted "run" of tables. + pub target_size: u64, +} + +/// Describes what to do (compact or not) +#[derive(Debug, Eq, PartialEq)] +pub enum Choice { + /// Just do nothing. + DoNothing, + + /// Moves tables into another level without rewriting. + Move(Input), + + /// Compacts some tables into a new level. + Merge(Input), + + /// Delete tables without doing compaction. + /// + /// This may be used by a compaction strategy that wants to delete old data + /// without having to compact it away, like [`fifo::Strategy`]. + Drop(HashSet), +} + +/// Trait for a compaction strategy +/// +/// The strategy receives the levels of the LSM-tree as argument +/// and emits a choice on what to do. +#[expect(clippy::module_name_repetitions)] +pub trait CompactionStrategy { + /// Gets the compaction strategy name. + fn get_name(&self) -> &'static str; + + #[doc(hidden)] + fn get_config(&self) -> Vec { + vec![] + } + + /// Decides on what to do based on the current state of the LSM-tree's levels + fn choose(&self, version: &Version, config: &Config, state: &CompactionState) -> Choice; +} diff --git a/crates/lsm-tree/src/compaction/movedown.rs b/crates/lsm-tree/src/compaction/movedown.rs new file mode 100644 index 000000000..2b9902a88 --- /dev/null +++ b/crates/lsm-tree/src/compaction/movedown.rs @@ -0,0 +1,38 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy, Input}; +use crate::{compaction::state::CompactionState, table::Table, version::Version, Config}; + +/// Moves down a level into the destination level. +pub struct Strategy(pub u8, pub u8); + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + "MoveDownCompaction" + } + + fn choose(&self, version: &Version, _: &Config, state: &CompactionState) -> Choice { + if version.level_is_busy(usize::from(self.0), state.hidden_set()) { + return Choice::DoNothing; + } + + let Some(level) = version.level(self.0.into()) else { + return Choice::DoNothing; + }; + + let table_ids = level + .iter() + .flat_map(|run| run.iter()) + .map(Table::id) + .collect(); + + Choice::Move(Input { + table_ids, + dest_level: self.1, + canonical_level: self.1, + target_size: u64::MAX, + }) + } +} diff --git a/crates/lsm-tree/src/compaction/pulldown.rs b/crates/lsm-tree/src/compaction/pulldown.rs new file mode 100644 index 000000000..09a558fd5 --- /dev/null +++ b/crates/lsm-tree/src/compaction/pulldown.rs @@ -0,0 +1,42 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy}; +use crate::{ + compaction::{state::CompactionState, Input}, + version::Version, + Config, +}; + +/// Pulls down and merges a level into the destination level. +/// +/// Used for unit tests. +pub struct Strategy(pub u8, pub u8); + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + "PullDownCompaction" + } + + #[expect(clippy::expect_used)] + fn choose(&self, version: &Version, _: &Config, _: &CompactionState) -> Choice { + let level = version + .level(usize::from(self.0)) + .expect("source level should exist"); + + let next_level = version + .level(usize::from(self.1)) + .expect("destination level should exist"); + + let mut table_ids = level.list_ids(); + table_ids.extend(next_level.list_ids()); + + Choice::Merge(Input { + table_ids, + dest_level: self.1, + target_size: 64_000_000, + canonical_level: 6, // We don't really care - this compaction is only used for very specific unit tests + }) + } +} diff --git a/crates/lsm-tree/src/compaction/state/hidden_set.rs b/crates/lsm-tree/src/compaction/state/hidden_set.rs new file mode 100644 index 000000000..6b3d55739 --- /dev/null +++ b/crates/lsm-tree/src/compaction/state/hidden_set.rs @@ -0,0 +1,47 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::TableId; + +/// The hidden set keeps track of which tables are currently being compacted +/// +/// When a table is hidden (being compacted), no other compaction task can include that +/// table, or it will be declined to be run. +/// +/// If a compaction task fails, the tables are shown again (removed from the hidden set). +#[derive(Clone, Default)] +pub struct HiddenSet { + pub(crate) set: crate::HashSet, +} + +impl HiddenSet { + pub(crate) fn hide>(&mut self, keys: T) { + self.set.extend(keys); + } + + pub(crate) fn show>(&mut self, keys: T) { + for key in keys { + self.set.remove(&key); + } + } + + pub(crate) fn is_blocked>(&self, ids: T) -> bool { + ids.into_iter().any(|id| self.is_hidden(id)) + } + + pub(crate) fn is_hidden(&self, key: TableId) -> bool { + self.set.contains(&key) + } + + pub(crate) fn is_empty(&self) -> bool { + self.set.is_empty() + } + + pub(crate) fn should_decline_compaction>( + &self, + ids: T, + ) -> bool { + self.is_blocked(ids) + } +} diff --git a/crates/lsm-tree/src/compaction/state/mod.rs b/crates/lsm-tree/src/compaction/state/mod.rs new file mode 100644 index 000000000..3a4d10fce --- /dev/null +++ b/crates/lsm-tree/src/compaction/state/mod.rs @@ -0,0 +1,90 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod hidden_set; + +use hidden_set::HiddenSet; + +#[derive(Default)] +pub struct CompactionState { + /// Set of table IDs that are masked. + /// + /// While consuming tables (because of compaction) they will not appear in the list of tables + /// as to not cause conflicts between multiple compaction threads (compacting the same tables). + hidden_set: HiddenSet, +} + +impl CompactionState { + pub fn hidden_set(&self) -> &HiddenSet { + &self.hidden_set + } + + pub fn hidden_set_mut(&mut self) -> &mut HiddenSet { + &mut self.hidden_set + } +} + +#[cfg(test)] +#[expect(clippy::expect_used)] +mod tests { + use crate::{AbstractTree, SequenceNumberCounter}; + use test_log::test; + + #[test] + #[ignore = "wip"] + fn level_manifest_atomicity() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + tree.insert("a", "a", 1); + tree.flush_active_memtable(0)?; + tree.insert("a", "a", 2); + tree.flush_active_memtable(0)?; + + assert_eq!(3, tree.approximate_len()); + + tree.major_compact(u64::MAX, 3)?; + + assert_eq!(1, tree.table_count()); + + tree.insert("a", "a", 3); + tree.flush_active_memtable(0)?; + + let table_count_before_major_compact = tree.table_count(); + + let crate::AnyTree::Standard(tree) = tree else { + unreachable!(); + }; + + // { + // // NOTE: Purposefully change level manifest to have invalid path + // // to force an I/O error + // tree.compaction_state + // .lock() + // .expect("lock is poisoned") + // .folder = "/invaliiid/asd".into(); + // } + + assert!(tree.major_compact(u64::MAX, 4).is_err()); + + assert!(tree + .compaction_state + .lock() + .expect("lock is poisoned") + .hidden_set() + .is_empty()); + + assert_eq!(table_count_before_major_compact, tree.table_count()); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/compaction/stream.rs b/crates/lsm-tree/src/compaction/stream.rs new file mode 100644 index 000000000..45a63e4c9 --- /dev/null +++ b/crates/lsm-tree/src/compaction/stream.rs @@ -0,0 +1,835 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{InternalValue, SeqNo, UserKey, UserValue, ValueType}; +use std::iter::Peekable; + +type Item = crate::Result; + +/// A callback that receives all dropped KVs +/// +/// Used for counting blobs that are not referenced anymore because of +/// vHandles that are being dropped through compaction. +pub trait DroppedKvCallback { + fn on_dropped(&mut self, kv: &InternalValue); +} + +/// Verdict returned by [`StreamFilter`] +#[derive(Debug)] +pub enum StreamFilterVerdict { + /// Keep the item as is. + Keep, + + /// Replace the item. + Replace((ValueType, UserValue)), + + /// Drop the item without leaving a tombstone. + Drop, +} + +/// A callback for modifying KVs in the stream +pub trait StreamFilter { + /// Handle an item, possibly modifying it. + fn filter_item(&mut self, item: &InternalValue) -> crate::Result; +} + +/// A [`StreamFilter`] that does not modify anything +pub struct NoFilter; + +impl StreamFilter for NoFilter { + fn filter_item(&mut self, _item: &InternalValue) -> crate::Result { + Ok(StreamFilterVerdict::Keep) + } +} + +/// Consumes a stream of KVs and emits a new stream according to GC and tombstone rules +/// +/// This iterator is used during flushing & compaction. +pub struct CompactionStream<'a, I: Iterator, F: StreamFilter = NoFilter> { + /// KV stream + inner: Peekable, + + /// MVCC watermark to get rid of old versions + gc_seqno_threshold: SeqNo, + + /// Event emitter that receives all dropped KVs + dropped_callback: Option<&'a mut dyn DroppedKvCallback>, + + /// Stream filter + filter: F, + + evict_tombstones: bool, + + zero_seqnos: bool, +} + +impl> CompactionStream<'_, I, NoFilter> { + /// Initializes a new merge iterator + #[must_use] + pub fn new(iter: I, gc_seqno_threshold: SeqNo) -> Self { + let iter = iter.peekable(); + + Self { + inner: iter, + gc_seqno_threshold, + dropped_callback: None, + filter: NoFilter, + evict_tombstones: false, + zero_seqnos: false, + } + } +} + +impl<'a, I: Iterator, F: StreamFilter + 'a> CompactionStream<'a, I, F> { + /// Installs a filter into this stream. + pub fn with_filter(self, filter: NF) -> CompactionStream<'a, I, NF> { + CompactionStream { + inner: self.inner, + gc_seqno_threshold: self.gc_seqno_threshold, + dropped_callback: self.dropped_callback, + filter, + evict_tombstones: self.evict_tombstones, + zero_seqnos: self.zero_seqnos, + } + } + + pub fn evict_tombstones(mut self, b: bool) -> Self { + self.evict_tombstones = b; + self + } + + /// Installs a callback that receives all dropped KVs. + pub fn with_drop_callback(mut self, cb: &'a mut dyn DroppedKvCallback) -> Self { + self.dropped_callback = Some(cb); + self + } + + /// Sets sequence numbers to zero if they are below the snapshot watermark. + /// + /// This can save a lot of space, because "0" only takes 1 byte, and sequence numbers are monotonically increasing. + pub fn zero_seqnos(mut self, b: bool) -> Self { + self.zero_seqnos = b; + self + } + + /// Drains the remaining versions of the given key. + fn drain_key(&mut self, key: &UserKey) -> crate::Result<()> { + loop { + let Some(next) = self.inner.next_if(|kv| { + if let Ok(kv) = kv { + let expired = kv.key.user_key == key; + + if expired { + if let Some(watcher) = &mut self.dropped_callback { + watcher.on_dropped(kv); + } + } + + expired + } else { + true + } + }) else { + return Ok(()); + }; + + next?; + } + } +} + +impl<'a, I: Iterator, F: StreamFilter + 'a> Iterator for CompactionStream<'a, I, F> { + type Item = Item; + + fn next(&mut self) -> Option { + loop { + let mut head = fail_iter!(self.inner.next()?); + + if !head.is_tombstone() { + match fail_iter!(self.filter.filter_item(&head)) { + StreamFilterVerdict::Keep => { /* Do nothing */ } + StreamFilterVerdict::Replace((new_type, new_value)) => { + // If we are replacing this item's value, call the dropped callback for the previous item + if let Some(watcher) = &mut self.dropped_callback { + watcher.on_dropped(&head); + } + head.value = new_value; + head.key.value_type = new_type; + } + StreamFilterVerdict::Drop => { + if let Some(watcher) = &mut self.dropped_callback { + watcher.on_dropped(&head); + } + + // Ignore + continue; + } + } + } + + if let Some(peeked) = self.inner.peek() { + let Ok(peeked) = peeked else { + #[expect( + clippy::expect_used, + reason = "we just asserted, the peeked value is an error" + )] + return Some(Err(self + .inner + .next() + .expect("value should exist") + .expect_err("should be error"))); + }; + + if peeked.key.user_key > head.key.user_key { + if head.is_tombstone() && self.evict_tombstones { + continue; + } + + // NOTE: Only item of this key and thus latest version, so return it no matter what + // ... + } else if peeked.key.seqno < self.gc_seqno_threshold { + if head.key.value_type == ValueType::Tombstone && self.evict_tombstones { + fail_iter!(self.drain_key(&head.key.user_key)); + continue; + } + + // NOTE: If next item is an actual value, and current value is weak tombstone, + // drop the tombstone + let drop_weak_tombstone = peeked.key.value_type == ValueType::Value + && head.key.value_type == ValueType::WeakTombstone; + + // NOTE: Next item is expired, + // so the tail of this user key is entirely expired, so drain it all + fail_iter!(self.drain_key(&head.key.user_key)); + + if drop_weak_tombstone { + continue; + } + } + } else if head.is_tombstone() && self.evict_tombstones { + continue; + } + + if self.zero_seqnos && head.key.seqno < self.gc_seqno_threshold { + head.key.seqno = 0; + } + + return Some(Ok(head)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{value::InternalValue, ValueType}; + use test_log::test; + + macro_rules! stream { + ($($key:expr, $sub_key:expr, $value_type:expr),* $(,)?) => {{ + let mut values = Vec::new(); + let mut counters = std::collections::HashMap::new(); + + $( + #[expect(clippy::string_lit_as_bytes)] + let key = $key.as_bytes(); + + #[expect(clippy::string_lit_as_bytes)] + let sub_key = $sub_key.as_bytes(); + + let value_type = match $value_type { + "V" => ValueType::Value, + "T" => ValueType::Tombstone, + "W" => ValueType::WeakTombstone, + _ => panic!("Unknown value type"), + }; + + let counter = counters.entry($key).and_modify(|x| { *x -= 1 }).or_insert(999); + values.push(InternalValue::from_components(key, sub_key, *counter, value_type)); + )* + + values + }}; + } + + macro_rules! iter_closed { + ($iter:expr) => { + assert!($iter.next().is_none(), "iterator should be closed (done)"); + }; + } + + #[derive(Default)] + struct TrackCallback { + items: Vec, + } + + impl DroppedKvCallback for TrackCallback { + fn on_dropped(&mut self, kv: &InternalValue) { + self.items.push(kv.clone()); + } + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_expired_callback_1() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "T", + "a", "", "T", + "a", "", "T", + ]; + + let mut my_watcher = TrackCallback::default(); + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 1_000).with_drop_callback(&mut my_watcher); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + assert_eq!( + [ + InternalValue::from_components("a", "", 998, ValueType::Tombstone), + InternalValue::from_components("a", "", 997, ValueType::Tombstone), + ], + &*my_watcher.items, + ); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_seqno_zeroing_1() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "3", "V", + "a", "2", "V", + "a", "1", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 1_000).zero_seqnos(true); + + assert_eq!( + InternalValue::from_components(*b"a", *b"3", 0, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + fn compaction_stream_queue_weak_tombstones() { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "old", "V", + "b", "", "W", + "b", "old", "V", + "c", "", "W", + "c", "old", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 1_050); + + iter_closed!(iter); + } + + /// GC should not evict tombstones, unless they are covered up + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_tombstone_no_gc() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "T", + "b", "", "T", + "c", "", "T", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 1_000_000); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_old_tombstone() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "T", + "a", "", "T", + "b", "", "T", + "b", "", "T", + "c", "", "T", + "c", "", "T", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 998); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"a", *b"", 998, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 998, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 998, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_tombstone_overwrite_gc() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "val", "V", + "a", "", "T", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 999); + + assert_eq!( + InternalValue::from_components(*b"a", *b"val", 999, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_weak_tombstone_simple() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "old", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 0); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"a", *b"old", 998, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_weak_tombstone_no_gc() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "old", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 998); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"a", *b"old", 998, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + fn compaction_stream_weak_tombstone_evict() { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "old", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 999); + + // NOTE: Weak tombstone is consumed because value is GC'ed + + iter_closed!(iter); + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_weak_tombstone_evict_next_value() -> crate::Result<()> { + #[rustfmt::skip] + let mut vec = stream![ + "a", "", "W", + "a", "old", "V", + ]; + vec.push(InternalValue::from_components( + "b", + "other", + 999, + ValueType::Value, + )); + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 999); + + // NOTE: Weak tombstone is consumed because value is GC'ed + + assert_eq!( + InternalValue::from_components(*b"b", *b"other", 999, ValueType::Value), + iter.next().unwrap()?, + ); + + iter_closed!(iter); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_no_evict_simple() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "old", "V", + "b", "old", "V", + "c", "old", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 0); + + assert_eq!( + InternalValue::from_components(*b"a", *b"old", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"old", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"old", 999, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_no_evict_simple_multi_keys() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "new", "V", + "a", "old", "V", + "b", "new", "V", + "b", "old", "V", + "c", "newnew", "V", + "c", "new", "V", + "c", "old", "V", + ]; + + let iter = vec.iter().cloned().map(Ok); + let mut iter = CompactionStream::new(iter, 0); + + assert_eq!( + InternalValue::from_components(*b"a", *b"new", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"a", *b"old", 998, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"new", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"old", 998, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"newnew", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"new", 998, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"old", 997, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + Ok(()) + } + + #[test] + fn compaction_stream_filter_1() { + struct Filter(&'static [u8]); + impl StreamFilter for Filter { + fn filter_item(&mut self, value: &InternalValue) -> crate::Result { + if value.key.user_key == b"b" { + Ok(StreamFilterVerdict::Drop) + } else if value.value < self.0 { + Ok(StreamFilterVerdict::Replace(( + ValueType::Tombstone, + UserValue::empty(), + ))) + } else { + Ok(StreamFilterVerdict::Keep) + } + } + } + + #[rustfmt::skip] + let vec = stream![ + "a", "9", "V", + "a", "8", "V", + "a", "7", "V", + // subsequent values will be filtered out + "a", "6", "V", + "a", "5", "V", + // subsequent values below gc threshold after filter + "a", "4", "V", + + // this value will be dropped without leaving a tombstone + "b", "b", "V", + ]; + + let mut drop_cb = TrackCallback { items: vec![] }; + let iter = vec.iter().cloned().map(Ok); + let iter = CompactionStream::new(iter, 995) + .with_filter(Filter(b"7")) + .with_drop_callback(&mut drop_cb); + + let out: Vec<_> = iter.map(Result::unwrap).collect(); + + #[rustfmt::skip] + assert_eq!(out, stream![ + "a", "9", "V", + "a", "8", "V", + "a", "7", "V", + "a", "", "T", + "a", "", "T", + ]); + + let fc = InternalValue::from_components; + + #[rustfmt::skip] + assert_eq!(drop_cb.items, [ + fc(b"a", b"6", 996, ValueType::Value), + fc(b"a", b"5", 995, ValueType::Value), + fc(b"a", b"4", 994, ValueType::Value), + fc(b"b", b"b", 999, ValueType::Value), + ]); + } + + pub mod custom_mvcc { + use super::*; + use byteorder::{ReadBytesExt, WriteBytesExt, BE}; + use test_log::test; + + /// MVCC trailer size (anything but user key) + const TRAILER_SIZE: usize = 10; + + // Our keys become a multi map of: # + // + // (type does not really matter for ordering, because key+seqno are unique anyway) + fn kv(key: &[u8], seqno: SeqNo, value: &[u8], tomb: bool) -> InternalValue { + InternalValue::from_components( + { + use std::io::Write; + + let len = key.len() + TRAILER_SIZE; + + let mut key_builder = unsafe { UserKey::builder_unzeroed(len) }; + let mut cursor = std::io::Cursor::new(&mut key_builder[..]); + + cursor.write_all(key).unwrap(); + cursor.write_u8(0).unwrap(); // Keys are variable size so we need a \0 delimiter + cursor + .write_u64::( + // IMPORTANT: Invert the seqno for correct descending sort + !seqno, + ) + .unwrap(); + cursor.write_u8(if tomb { 1 } else { 0 }).unwrap(); + + debug_assert_eq!(len, cursor.position() as usize); + + key_builder.freeze() + }, + value, + 2_353, // does not matter for us + ValueType::Value, + ) + } + + struct Filter { + /// The previous user key + /// + /// Note that the user key is NOT the full KV key + /// because we embed MVCC information into the key (user_key#seqno#type). + prev_user_key: Option, + + /// MVCC watermark we can safely delete if an item < watermark + /// is covered by a newer version. + mvcc_watermark: SeqNo, + } + + impl StreamFilter for Filter { + fn filter_item(&mut self, value: &InternalValue) -> crate::Result { + let l = value.key.user_key.len(); + + // User key len + let ukl = l - TRAILER_SIZE; + + match &self.prev_user_key { + Some(prev) => { + let user_key = &value.key.user_key[..ukl]; + + if prev == &user_key { + // We found another, older version of the previous key + let mut seqno = &value.key.user_key[(ukl + 1)..l - 1]; + debug_assert_eq!(8, seqno.len()); + + // IMPORTANT: Invert the seqno back to normal value + let seqno = !seqno.read_u64::().unwrap(); + + if seqno < self.mvcc_watermark { + return Ok(StreamFilterVerdict::Drop); + } + } else { + let user_key = &value.key.user_key.slice(..ukl); + self.prev_user_key = Some(user_key.clone()); + } + } + None => { + let user_key = &value.key.user_key.slice(..ukl); + self.prev_user_key = Some(user_key.clone()); + } + } + + Ok(StreamFilterVerdict::Keep) + } + } + + #[test] + fn compaction_filter_custom_mvcc() { + let vec = vec![ + kv(b"abc", 4, b"c", false), + kv(b"abc", 3, b"b", false), + kv(b"abc", 2, b"a", false), + ]; + + let mut drop_cb = TrackCallback { items: vec![] }; + let iter = vec.iter().cloned().map(Ok); + let iter = CompactionStream::new(iter, 995) + .with_filter(Filter { + mvcc_watermark: 5, + prev_user_key: None, + }) + .with_drop_callback(&mut drop_cb); + + let out: Vec<_> = iter.map(Result::unwrap).collect(); + + #[rustfmt::skip] + assert_eq!(out, vec![ + kv(b"abc", 4, b"c", false), + ]); + } + + #[test] + fn compaction_filter_custom_mvcc_multi_keys() { + let vec = vec![ + kv(b"a", 4, b"c", false), + kv(b"a", 3, b"b", false), + kv(b"a", 2, b"a", false), + // + kv(b"b", 4, b"c", false), + kv(b"b", 3, b"b", false), + kv(b"b", 2, b"a", false), + // + kv(b"c", 1, b"c", false), + // + kv(b"d", 0, b"c", false), + ]; + + let mut drop_cb = TrackCallback { items: vec![] }; + let iter = vec.iter().cloned().map(Ok); + let iter = CompactionStream::new(iter, 995) + .with_filter(Filter { + mvcc_watermark: 3, + prev_user_key: None, + }) + .with_drop_callback(&mut drop_cb); + + let out: Vec<_> = iter.map(Result::unwrap).collect(); + + #[rustfmt::skip] + assert_eq!(out, vec![ + kv(b"a", 4, b"c", false), + kv(b"a", 3, b"b", false), + // + kv(b"b", 4, b"c", false), + kv(b"b", 3, b"b", false), + // + kv(b"c", 1, b"c", false), + // + kv(b"d", 0, b"c", false), + ]); + } + } +} diff --git a/crates/lsm-tree/src/compaction/tiered.rs b/crates/lsm-tree/src/compaction/tiered.rs new file mode 100644 index 000000000..7ee618766 --- /dev/null +++ b/crates/lsm-tree/src/compaction/tiered.rs @@ -0,0 +1,343 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Choice, CompactionStrategy}; +use crate::{compaction::state::CompactionState, version::Version, Config}; + +// fn desired_level_size_in_bytes(level_idx: u8, ratio: u8, base_size: u32) -> usize { +// (ratio as usize).pow(u32::from(level_idx + 1)) * (base_size as usize) +// } + +/// Size-tiered compaction strategy (STCS) +/// +/// If a level reaches a threshold, it is merged into a larger table to the next level. +/// +/// STCS suffers from high read and temporary doubled space amplification, but has good write amplification. +#[derive(Clone)] +pub struct Strategy { + /// Base size + pub base_size: u32, + + /// Size ratio between levels of the LSM tree (a.k.a fanout, growth rate). + /// + /// This is the exponential growth of the from one + /// level to the next + /// + /// A level target size is: base_size * level_ratio.pow(#level + 1) + #[expect(clippy::doc_markdown)] + pub level_ratio: u8, +} + +impl Strategy { + /// Creates a new STCS strategy with custom base size + #[must_use] + pub fn new(base_size: u32, level_ratio: u8) -> Self { + Self { + base_size, + level_ratio, + } + } +} + +impl Default for Strategy { + fn default() -> Self { + Self { + base_size: 64 * 1_024 * 1_024, + level_ratio: 4, + } + } +} + +impl CompactionStrategy for Strategy { + fn get_name(&self) -> &'static str { + "TieredStrategy" + } + + fn choose(&self, version: &Version, _config: &Config, state: &CompactionState) -> Choice { + unimplemented!() + } +} +/* +#[cfg(test)] +mod tests { + use super::Strategy; + use crate::{ + cache::Cache, + compaction::{Choice, CompactionStrategy, Input as CompactionInput}, + config::Config, + descriptor_table::FileDescriptorTable, + file::LEVELS_MANIFEST_FILE, + level_manifest::LevelManifest, + segment::{ + block::offset::BlockOffset, + block_index::{two_level_index::TwoLevelBlockIndex, BlockIndexImpl}, + file_offsets::FileOffsets, + meta::{Metadata, SegmentId}, + SegmentInner, + }, + super_segment::Segment, + HashSet, KeyRange, SeqNo, + }; + use std::sync::{atomic::AtomicBool, Arc}; + use test_log::test; + + #[allow(clippy::expect_used)] + fn fixture_segment(id: SegmentId, size_mib: u64, max_seqno: SeqNo) -> Segment { + todo!() + + /* let cache = Arc::new(Cache::with_capacity_bytes(10 * 1_024 * 1_024)); + + let block_index = TwoLevelBlockIndex::new((0, id).into(), cache.clone()); + let block_index = Arc::new(BlockIndexImpl::TwoLevel(block_index)); + + SegmentInner { + tree_id: 0, + descriptor_table: Arc::new(FileDescriptorTable::new(512, 1)), + block_index, + + offsets: FileOffsets { + bloom_ptr: BlockOffset(0), + range_filter_ptr: BlockOffset(0), + index_block_ptr: BlockOffset(0), + metadata_ptr: BlockOffset(0), + range_tombstones_ptr: BlockOffset(0), + tli_ptr: BlockOffset(0), + pfx_ptr: BlockOffset(0), + }, + + metadata: Metadata { + data_block_count: 0, + index_block_count: 0, + data_block_size: 4_096, + index_block_size: 4_096, + created_at: 0, + id, + file_size: size_mib * 1_024 * 1_024, + compression: crate::segment::meta::CompressionType::None, + table_type: crate::segment::meta::TableType::Block, + item_count: 0, + key_count: 0, + key_range: KeyRange::new((vec![].into(), vec![].into())), + tombstone_count: 0, + range_tombstone_count: 0, + uncompressed_size: size_mib * 1_024 * 1_024, + seqnos: (0, max_seqno), + }, + cache, + + bloom_filter: Some(BloomFilter::with_fp_rate(1, 0.1)), + + path: "a".into(), + is_deleted: AtomicBool::default(), + } + .into() */ + } + + #[test] + fn tiered_empty_levels() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 8, + }; + + let levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + + assert_eq!( + compactor.choose(&levels, &Config::default()), + Choice::DoNothing + ); + + Ok(()) + } + + #[test] + fn tiered_default_l0() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 4, + }; + let config = Config::default(); + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + + levels.add(fixture_segment(1, 8, 5)); + assert_eq!(compactor.choose(&levels, &config), Choice::DoNothing); + + levels.add(fixture_segment(2, 8, 6)); + assert_eq!(compactor.choose(&levels, &config), Choice::DoNothing); + + levels.add(fixture_segment(3, 8, 7)); + assert_eq!(compactor.choose(&levels, &config), Choice::DoNothing); + + levels.add(fixture_segment(4, 8, 8)); + + assert_eq!( + compactor.choose(&levels, &config), + Choice::Merge(CompactionInput { + dest_level: 1, + segment_ids: set![1, 2, 3, 4], + target_size: u64::MAX, + }) + ); + + Ok(()) + } + + #[test] + fn tiered_ordering() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 2, + }; + let config = Config::default(); + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + + levels.add(fixture_segment(1, 8, 0)); + levels.add(fixture_segment(2, 8, 1)); + levels.add(fixture_segment(3, 8, 2)); + levels.add(fixture_segment(4, 8, 3)); + + assert_eq!( + compactor.choose(&levels, &config), + Choice::Merge(CompactionInput { + dest_level: 1, + segment_ids: set![1, 2], + target_size: u64::MAX, + }) + ); + + Ok(()) + } + + #[test] + fn tiered_more_than_min() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 4, + }; + let config = Config::default(); + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + levels.add(fixture_segment(1, 8, 5)); + levels.add(fixture_segment(2, 8, 6)); + levels.add(fixture_segment(3, 8, 7)); + levels.add(fixture_segment(4, 8, 8)); + + levels.insert_into_level(1, fixture_segment(5, 8 * 4, 9)); + levels.insert_into_level(1, fixture_segment(6, 8 * 4, 10)); + levels.insert_into_level(1, fixture_segment(7, 8 * 4, 11)); + levels.insert_into_level(1, fixture_segment(8, 8 * 4, 12)); + + assert_eq!( + compactor.choose(&levels, &config), + Choice::Merge(CompactionInput { + dest_level: 2, + segment_ids: set![5, 6, 7, 8], + target_size: u64::MAX, + }) + ); + + Ok(()) + } + + #[test] + fn tiered_many_segments() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 2, + }; + let config = Config::default(); + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + levels.add(fixture_segment(1, 8, 5)); + levels.add(fixture_segment(2, 8, 6)); + levels.add(fixture_segment(3, 8, 7)); + levels.add(fixture_segment(4, 8, 8)); + + assert_eq!( + compactor.choose(&levels, &config), + Choice::Merge(CompactionInput { + dest_level: 1, + segment_ids: set![1, 2], + target_size: u64::MAX, + }) + ); + + Ok(()) + } + + #[test] + fn tiered_deeper_level() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 2, + }; + let config = Config::default(); + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + levels.add(fixture_segment(1, 8, 5)); + + levels.insert_into_level(1, fixture_segment(2, 8 * 2, 6)); + levels.insert_into_level(1, fixture_segment(3, 8 * 2, 7)); + + assert_eq!( + compactor.choose(&levels, &config), + Choice::Merge(CompactionInput { + dest_level: 2, + segment_ids: set![2, 3], + target_size: u64::MAX, + }) + ); + + let tempdir = tempfile::tempdir()?; + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + + levels.insert_into_level(2, fixture_segment(2, 8 * 4, 5)); + levels.insert_into_level(2, fixture_segment(3, 8 * 4, 6)); + + assert_eq!( + compactor.choose(&levels, &config), + Choice::Merge(CompactionInput { + dest_level: 3, + segment_ids: set![2, 3], + target_size: u64::MAX, + }) + ); + + Ok(()) + } + + #[test] + fn tiered_last_level() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + + let compactor = Strategy { + base_size: 8 * 1_024 * 1_024, + level_ratio: 2, + }; + let config = Config::default(); + + let mut levels = LevelManifest::create_new(4, tempdir.path().join(LEVELS_MANIFEST_FILE))?; + levels.insert_into_level(3, fixture_segment(2, 8, 5)); + levels.insert_into_level(3, fixture_segment(3, 8, 5)); + + assert_eq!(compactor.choose(&levels, &config), Choice::DoNothing); + + Ok(()) + } +} + */ diff --git a/crates/lsm-tree/src/compaction/worker.rs b/crates/lsm-tree/src/compaction/worker.rs new file mode 100644 index 000000000..da8d88f14 --- /dev/null +++ b/crates/lsm-tree/src/compaction/worker.rs @@ -0,0 +1,960 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{CompactionStrategy, Input as CompactionPayload}; +use crate::{ + blob_tree::FragmentationMap, + compaction::{ + filter::{Context, StreamFilterAdapter}, + flavour::{RelocatingCompaction, StandardCompaction}, + state::CompactionState, + stream::CompactionStream, + Choice, + }, + file::BLOBS_FOLDER, + merge::Merger, + run_scanner::RunScanner, + stop_signal::StopSignal, + tree::inner::TreeId, + version::{Run, SuperVersions, Version}, + vlog::{BlobFileMergeScanner, BlobFileScanner, BlobFileWriter}, + BlobFile, Config, HashSet, InternalValue, SeqNo, SequenceNumberCounter, Table, TableId, +}; +use std::{ + sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard}, + time::Instant, +}; + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +pub type CompactionReader<'a> = Box> + 'a>; + +/// Compaction options +pub struct Options { + pub tree_id: TreeId, + + pub global_seqno: SequenceNumberCounter, + + pub visible_seqno: SequenceNumberCounter, + + pub table_id_generator: SequenceNumberCounter, + + pub blob_file_id_generator: SequenceNumberCounter, + + /// Configuration of tree. + pub config: Arc, + + pub version_history: Arc>, + + /// Compaction strategy to use. + pub strategy: Arc, + + /// Stop signal to interrupt a compaction worker in case + /// the tree is dropped. + pub stop_signal: StopSignal, + + /// Evicts items that are older than this seqno (MVCC GC). + pub mvcc_gc_watermark: u64, + + pub compaction_state: Arc>, + + #[cfg(feature = "metrics")] + pub metrics: Arc, +} + +impl Options { + pub fn from_tree(tree: &crate::Tree, strategy: Arc) -> Self { + Self { + global_seqno: tree.config.seqno.clone(), + visible_seqno: tree.config.visible_seqno.clone(), + tree_id: tree.id, + table_id_generator: tree.table_id_counter.clone(), + blob_file_id_generator: tree.blob_file_id_counter.clone(), + config: tree.config.clone(), + version_history: tree.version_history.clone(), + stop_signal: tree.stop_signal.clone(), + strategy, + mvcc_gc_watermark: 0, + + compaction_state: tree.compaction_state.clone(), + + #[cfg(feature = "metrics")] + metrics: tree.metrics.clone(), + } + } +} + +/// Runs compaction task. +/// +/// This will block until the compactor is fully finished. +pub fn do_compaction(opts: &Options) -> crate::Result<()> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let compaction_state = opts.compaction_state.lock().expect("lock is poisoned"); + + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let version_history_lock = opts.version_history.read().expect("lock is poisoned"); + + let start = Instant::now(); + log::trace!( + "Consulting compaction strategy {:?}", + opts.strategy.get_name(), + ); + let choice = opts.strategy.choose( + &version_history_lock.latest_version().version, + &opts.config, + &compaction_state, + ); + + log::debug!("Compaction choice: {choice:?} in {:?}", start.elapsed()); + + match choice { + Choice::Merge(payload) => { + merge_tables(compaction_state, version_history_lock, opts, &payload) + } + Choice::Move(payload) => { + drop(version_history_lock); + + move_tables(&compaction_state, opts, &payload) + } + Choice::Drop(payload) => { + drop(version_history_lock); + + drop_tables( + compaction_state, + opts, + &payload.into_iter().collect::>(), + ) + } + Choice::DoNothing => { + log::trace!("Compactor chose to do nothing"); + Ok(()) + } + } +} + +fn pick_run_indexes(run: &Run
, to_compact: &[TableId]) -> Option<(usize, usize)> { + let lo = run + .iter() + .position(|table| to_compact.contains(&table.id()))?; + + let hi = run + .iter() + .rposition(|table| to_compact.contains(&table.id()))?; + + Some((lo, hi)) +} + +fn create_compaction_stream<'a>( + version: &Version, + to_compact: &[TableId], + eviction_seqno: SeqNo, +) -> crate::Result>>>> { + let mut readers: Vec> = vec![]; + let mut found = 0; + + for run in version.iter_levels().flat_map(|lvl| lvl.iter()) { + if run.len() > 1 { + let Some((lo, hi)) = pick_run_indexes(run, to_compact) else { + continue; + }; + + readers.push(Box::new(RunScanner::culled( + run.clone(), + (Some(lo), Some(hi)), + )?)); + + found += hi - lo + 1; + } else { + for table in run.iter().filter(|x| to_compact.contains(&x.metadata.id)) { + found += 1; + readers.push(Box::new(table.scan()?)); + } + } + } + + Ok(if found == to_compact.len() { + Some(CompactionStream::new(Merger::new(readers), eviction_seqno)) + } else { + None + }) +} + +fn move_tables( + compaction_state: &MutexGuard<'_, CompactionState>, + opts: &Options, + payload: &CompactionPayload, +) -> crate::Result<()> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_history_lock = opts.version_history.write().expect("lock is poisoned"); + + // Fail-safe for buggy compaction strategies + if compaction_state + .hidden_set() + .should_decline_compaction(payload.table_ids.iter().copied()) + { + log::warn!( + "Compaction task created by {:?} contained hidden tables, declining to run it - please report this at https://github.com/fjall-rs/lsm-tree/issues/new?template=bug_report.md", + opts.strategy.get_name(), + ); + return Ok(()); + } + + let table_ids = payload.table_ids.iter().copied().collect::>(); + + version_history_lock.upgrade_version( + &opts.config.path, + |current| { + let mut copy = current.clone(); + + copy.version = copy + .version + .with_moved(&table_ids, payload.dest_level as usize); + + Ok(copy) + }, + &opts.global_seqno, + &opts.visible_seqno, + )?; + + if let Err(e) = version_history_lock.maintenance(&opts.config.path, opts.mvcc_gc_watermark) { + log::error!("Manifest maintenance failed: {e:?}"); + return Err(e); + } + + Ok(()) +} + +/// Picks blob files to rewrite (defragment) +fn pick_blob_files_to_rewrite( + picked_tables: &HashSet, + current_version: &Version, + blob_opts: &crate::KvSeparationOptions, +) -> crate::Result> { + use crate::Table; + + // We start off by getting all the blob files that are referenced by the tables + // that we want to compact. + let linked_blob_files = picked_tables + .iter() + .map(|&id| { + current_version.get_table(id).unwrap_or_else(|| { + panic!("Table {id} should exist"); + }) + }) + .map(Table::list_blob_file_references) + .collect::, _>>()?; + + // Then we filter all blob files that are not fragmented or old enough. + let mut linked_blob_files = linked_blob_files + .into_iter() + .flatten() + .flatten() + .map(|blob_file_ref| { + current_version + .blob_files + .get(blob_file_ref.blob_file_id) + .unwrap_or_else(|| { + panic!("Blob file {} should exist", blob_file_ref.blob_file_id); + }) + }) + .filter(|blob_file| { + blob_file.is_stale(current_version.gc_stats(), blob_opts.staleness_threshold) + }) + .filter(|blob_file| { + // NOTE: Dead blob files are dropped anyway during current_version change commit + !blob_file.is_dead(current_version.gc_stats()) + }) + .collect::>() + .into_iter() + .collect::>(); + + linked_blob_files.sort_by_key(|a| a.id()); + + #[expect( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "precision loss and truncation are acceptable for cutoff calculation" + )] + let cutoff_point = { + let len = linked_blob_files.len() as f32; + (len * blob_opts.age_cutoff) as usize + }; + linked_blob_files.drain(cutoff_point..); + + // IMPORTANT: Additionally, we also have to check if any other tables reference any of our candidate blob files. + // We have to *not* include blob files that are referenced by other tables, because otherwise those + // blob references would point into nothing (becoming dangling). + for table in current_version.iter_tables() { + if picked_tables.contains(&table.id()) { + continue; + } + + let other_refs = table + .list_blob_file_references()? + .unwrap_or_default() + .into_iter() + .filter(|x| linked_blob_files.iter().any(|bf| bf.id() == x.blob_file_id)) + .collect::>(); + + for additional_ref in other_refs { + linked_blob_files.retain(|x| x.id() != additional_ref.blob_file_id); + } + } + + Ok(linked_blob_files.into_iter().cloned().collect::>()) +} + +fn hidden_guard( + payload: &CompactionPayload, + opts: &Options, + f: impl FnOnce() -> crate::Result, +) -> crate::Result { + f().inspect_err(|e| { + log::error!("Compaction failed: {e:?}"); + + // IMPORTANT: We need to show tables again on error + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut compaction_state = opts.compaction_state.lock().expect("lock is poisoned"); + + compaction_state + .hidden_set_mut() + .show(payload.table_ids.iter().copied()); + }) +} + +#[expect(clippy::too_many_lines)] +fn merge_tables( + mut compaction_state: MutexGuard<'_, CompactionState>, + version_history_lock: RwLockReadGuard<'_, SuperVersions>, + opts: &Options, + payload: &CompactionPayload, +) -> crate::Result<()> { + if opts.stop_signal.is_stopped() { + log::debug!("Stopping before compaction because of stop signal"); + return Ok(()); + } + + // Fail-safe for buggy compaction strategies + if compaction_state + .hidden_set() + .should_decline_compaction(payload.table_ids.iter().copied()) + { + log::warn!( + "Compaction task created by {:?} contained hidden tables, declining to run it - please report this at https://github.com/fjall-rs/lsm-tree/issues/new?template=bug_report.md", + opts.strategy.get_name(), + ); + drop(version_history_lock); + return Ok(()); + } + + let current_super_version = version_history_lock.latest_version(); + + let Some(tables) = payload + .table_ids + .iter() + .map(|&id| current_super_version.version.get_table(id).cloned()) + .collect::>>() + else { + log::warn!( + "Compaction task created by {:?} contained tables not referenced in the level manifest", + opts.strategy.get_name(), + ); + return Ok(()); + }; + + let mut blob_frag_map = FragmentationMap::default(); + + let Some(mut merge_iter) = create_compaction_stream( + ¤t_super_version.version, + &payload.table_ids.iter().copied().collect::>(), + opts.mvcc_gc_watermark, + )? + else { + log::warn!( + "Compaction task tried to compact tables that do not exist, declining to run it" + ); + return Ok(()); + }; + + let dst_lvl = payload.canonical_level.into(); + let last_level = opts.config.level_count - 1; + + // NOTE: Only evict tombstones when reaching the last level, + // That way we don't resurrect data beneath the tombstone + let is_last_level = payload.dest_level == last_level; + + merge_iter = merge_iter + .evict_tombstones(is_last_level) + .zero_seqnos(false); + + let blobs_folder = opts.config.path.join(BLOBS_FOLDER); + + let filter_ctx = Context { is_last_level }; + + // Construct the compaction filter + let mut compaction_filter = opts.config.compaction_filter_factory.as_ref().map(|f| { + log::trace!("Installing custom compaction filter {:?}", f.name()); + f.make_filter(&filter_ctx) + }); + + // This is used by the compaction filter if it wants to write new blobs + // TODO: the filter should really pipe new blobs into the compaction stream directly, + // TODO: but that will probably require to change the protocol between filter <-> compaction stream a bit + let mut filter_blob_writer = None; + let mut merge_iter = merge_iter.with_filter(StreamFilterAdapter::new( + compaction_filter.as_deref_mut(), + opts, + ¤t_super_version.version, + &blobs_folder, + &mut filter_blob_writer, + &filter_ctx, + )); + + let table_writer = + super::flavour::prepare_table_writer(¤t_super_version.version, opts, payload)?; + + let start = Instant::now(); + + let mut compactor = match &opts.config.kv_separation_opts { + Some(blob_opts) => { + merge_iter = merge_iter.with_drop_callback(&mut blob_frag_map); + + let blob_files_to_rewrite = pick_blob_files_to_rewrite( + &payload.table_ids, + ¤t_super_version.version, + blob_opts, + )?; + + if blob_files_to_rewrite.is_empty() { + log::debug!("No blob relocation needed"); + + Box::new(StandardCompaction::new(table_writer, tables)) + as Box + } else { + log::debug!( + "Relocate blob files: {:?}", + blob_files_to_rewrite + .iter() + .map(BlobFile::id) + .collect::>(), + ); + + let scanner = BlobFileMergeScanner::new( + blob_files_to_rewrite + .iter() + .map(|bf| BlobFileScanner::new(&bf.0.path, bf.id())) + .collect::>>()?, + ); + + let writer = BlobFileWriter::new( + opts.blob_file_id_generator.clone(), + &blobs_folder, + opts.tree_id, + opts.config.descriptor_table.clone(), + )? + .use_target_size(blob_opts.file_target_size) + .use_passthrough_compression(blob_opts.compression); + + let inner = StandardCompaction::new(table_writer, tables); + + Box::new(RelocatingCompaction::new( + inner, + scanner.peekable(), + writer, + blob_files_to_rewrite, + )) + } + } + None => Box::new(StandardCompaction::new(table_writer, tables)), + }; + + log::trace!("Blob file GC preparation done in {:?}", start.elapsed()); + + drop(version_history_lock); + + { + compaction_state + .hidden_set_mut() + .hide(payload.table_ids.iter().copied()); + } + + // IMPORTANT: Unlock exclusive compaction lock as we are now doing the actual (CPU-intensive) compaction + drop(compaction_state); + + hidden_guard(payload, opts, || { + for (idx, item) in merge_iter.enumerate() { + let item = item?; + + compactor.write(item)?; + + if idx % 1_000_000 == 0 && opts.stop_signal.is_stopped() { + log::debug!("Stopping amidst compaction because of stop signal"); + return Ok(()); + } + } + + Ok(()) + })?; + + if let Some(filter) = compaction_filter { + filter.finish(); + } + + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut compaction_state = opts.compaction_state.lock().expect("lock is poisoned"); + + log::trace!("Acquiring super version write lock"); + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_history_lock = opts.version_history.write().expect("lock is poisoned"); + log::trace!("Acquired super version write lock"); + + log::trace!("Blob fragmentation diff: {blob_frag_map:#?}"); + + let extra_blob_files = filter_blob_writer + .map(BlobFileWriter::finish) + .transpose() + .inspect_err(|e| { + // NOTE: We cannot use hidden_guard here because we already locked the compaction state + + log::error!("Compaction failed while finishing filter blob writer: {e:?}"); + + compaction_state + .hidden_set_mut() + .show(payload.table_ids.iter().copied()); + })? + .unwrap_or_default(); + + compactor + .finish( + &mut version_history_lock, + opts, + payload, + dst_lvl, + blob_frag_map, + extra_blob_files, + ) + .inspect_err(|e| { + // NOTE: We cannot use hidden_guard here because we already locked the compaction state + + log::error!("Compaction failed: {e:?}"); + + compaction_state + .hidden_set_mut() + .show(payload.table_ids.iter().copied()); + })?; + + compaction_state + .hidden_set_mut() + .show(payload.table_ids.iter().copied()); + + version_history_lock + .maintenance(&opts.config.path, opts.mvcc_gc_watermark) + .inspect_err(|e| { + log::error!("Manifest maintenance failed: {e:?}"); + })?; + + drop(version_history_lock); + drop(compaction_state); + + log::trace!("Compaction successful"); + + Ok(()) +} + +fn drop_tables( + compaction_state: MutexGuard<'_, CompactionState>, + opts: &Options, + ids_to_drop: &[TableId], +) -> crate::Result<()> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_history_lock = opts.version_history.write().expect("lock is poisoned"); + + // Fail-safe for buggy compaction strategies + if compaction_state + .hidden_set() + .should_decline_compaction(ids_to_drop.iter().copied()) + { + log::warn!( + "Compaction task created by {:?} contained hidden tables, declining to run it - please report this at https://github.com/fjall-rs/lsm-tree/issues/new?template=bug_report.md", + opts.strategy.get_name(), + ); + return Ok(()); + } + + let Some(tables) = ids_to_drop + .iter() + .map(|&id| { + version_history_lock + .latest_version() + .version + .get_table(id) + .cloned() + }) + .collect::>>() + else { + log::warn!( + "Compaction task created by {:?} contained tables not referenced in the level manifest", + opts.strategy.get_name(), + ); + return Ok(()); + }; + + log::debug!("Dropping tables: {ids_to_drop:?}"); + + let mut dropped_blob_files = vec![]; + + // IMPORTANT: Write the manifest with the removed tables first + // Otherwise the table files are deleted, but are still referenced! + version_history_lock.upgrade_version( + &opts.config.path, + |current| { + let mut copy = current.clone(); + + copy.version = copy + .version + .with_dropped(ids_to_drop, &mut dropped_blob_files)?; + + Ok(copy) + }, + &opts.global_seqno, + &opts.visible_seqno, + )?; + + if let Err(e) = version_history_lock.maintenance(&opts.config.path, opts.mvcc_gc_watermark) { + log::error!("Manifest maintenance failed: {e:?}"); + return Err(e); + } + + drop(version_history_lock); + + // NOTE: If the application were to crash >here< it's fine + // The tables are not referenced anymore, and will be + // cleaned up upon recovery + for table in tables { + table.mark_as_deleted(); + } + + for blob_file in dropped_blob_files { + blob_file.mark_as_deleted(); + } + + drop(compaction_state); + + log::trace!("Dropped {} tables", ids_to_drop.len()); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{create_compaction_stream, pick_run_indexes}; + use crate::{ + compaction::{state::CompactionState, Choice, CompactionStrategy, Input}, + config::BlockSizePolicy, + version::Version, + AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter, TableId, + }; + use std::sync::Arc; + use test_log::test; + + #[test] + fn compaction_stream_run_not_found() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + assert!(create_compaction_stream(&tree.current_version(), &[666], 0)?.is_none()); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_run() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + + assert_eq!( + Some((0, 2)), + pick_run_indexes( + tree.current_version() + .level(0) + .unwrap() + .iter() + .next() + .unwrap(), + &[0, 1, 2], + ) + ); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_run_2() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + + assert_eq!( + Some((0, 0)), + pick_run_indexes( + tree.current_version() + .level(0) + .unwrap() + .iter() + .next() + .unwrap(), + &[0], + ) + ); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_run_3() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + + assert_eq!( + Some((2, 2)), + pick_run_indexes( + tree.current_version() + .level(0) + .unwrap() + .iter() + .next() + .unwrap(), + &[2], + ) + ); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn compaction_stream_run_4() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + + assert_eq!( + None, + pick_run_indexes( + tree.current_version() + .level(0) + .unwrap() + .iter() + .next() + .unwrap(), + &[4], + ) + ); + + Ok(()) + } + + #[test] + fn compaction_drop_tables() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.approximate_len()); + assert_eq!(0, tree.sealed_memtable_count()); + + tree.insert("b", "a", 1); + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.approximate_len()); + assert_eq!(0, tree.sealed_memtable_count()); + + tree.insert("c", "a", 2); + tree.flush_active_memtable(0)?; + assert_eq!(3, tree.approximate_len()); + assert_eq!(0, tree.sealed_memtable_count()); + + tree.compact(Arc::new(crate::compaction::Fifo::new(1, None)), 3)?; + + assert_eq!(0, tree.table_count()); + + Ok(()) + } + + #[test] + fn blob_file_picking_simple() -> crate::Result<()> { + struct InPlaceStrategy(Vec); + + impl CompactionStrategy for InPlaceStrategy { + fn get_name(&self) -> &'static str { + "InPlaceCompaction" + } + + fn choose(&self, _: &Version, _: &Config, _: &CompactionState) -> Choice { + Choice::Merge(Input { + table_ids: self.0.iter().copied().collect(), + dest_level: 6, + target_size: 64_000_000, + canonical_level: 6, // We don't really care - this compaction is only used for very specific unit tests + }) + } + } + + let folder = tempfile::tempdir()?; + + let tree = crate::Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1)) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(1) + .age_cutoff(1.0) + .staleness_threshold(0.01) + .compression(crate::CompressionType::None), + )) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.flush_active_memtable(1_000)?; + assert_eq!(0, tree.sealed_memtable_count()); + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.major_compact(1, 1_000)?; + assert_eq!(3, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + // We now have tables [1, 2, 3] pointing into blob file 0 + + tree.drop_range("a"..="a")?; + assert_eq!(2, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + assert_eq!( + &{ + let mut map = crate::HashMap::default(); + map.insert(0, crate::blob_tree::FragmentationEntry::new(1, 1, 1)); + map + }, + &**tree.current_version().gc_stats(), + ); + } + + // Even though we are compacting table #2, blob file is not rewritten + // because table #3 still points into it + tree.compact(Arc::new(InPlaceStrategy(vec![2])), 1_000)?; + assert_eq!(2, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + assert_eq!( + &{ + let mut map = crate::HashMap::default(); + map.insert(0, crate::blob_tree::FragmentationEntry::new(1, 1, 1)); + map + }, + &**tree.current_version().gc_stats(), + ); + } + + // Because tables #3 & #4 both point into the blob file + // Only selecting both for compaction will actually rewrite the file + tree.compact(Arc::new(InPlaceStrategy(vec![3, 4])), 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + // Fragmentation is cleared up because blob file was relocated + { + assert_eq!( + crate::HashMap::default(), + **tree.current_version().gc_stats(), + ); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/compression.rs b/crates/lsm-tree/src/compression.rs new file mode 100644 index 000000000..4a2169537 --- /dev/null +++ b/crates/lsm-tree/src/compression.rs @@ -0,0 +1,94 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::coding::{Decode, Encode}; +use byteorder::{ReadBytesExt, WriteBytesExt}; +use std::io::{Read, Write}; + +/// Compression algorithm to use +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum CompressionType { + /// No compression + /// + /// Not recommended. + None, + + /// LZ4 compression + /// + /// Recommended for use cases with a focus + /// on speed over compression ratio. + #[cfg(feature = "lz4")] + Lz4, +} + +impl std::fmt::Display for CompressionType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::None => "none", + + #[cfg(feature = "lz4")] + Self::Lz4 => "lz4", + } + ) + } +} + +impl Encode for CompressionType { + fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error> { + match self { + Self::None => { + writer.write_u8(0)?; + } + + #[cfg(feature = "lz4")] + Self::Lz4 => { + writer.write_u8(1)?; + } + } + + Ok(()) + } +} + +impl Decode for CompressionType { + fn decode_from(reader: &mut R) -> Result { + let tag = reader.read_u8()?; + + match tag { + 0 => Ok(Self::None), + + #[cfg(feature = "lz4")] + 1 => Ok(Self::Lz4), + + tag => Err(crate::Error::InvalidTag(("CompressionType", tag))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn compression_serialize_none() { + let serialized = CompressionType::None.encode_into_vec(); + assert_eq!(1, serialized.len()); + } + + #[cfg(feature = "lz4")] + mod lz4 { + use super::*; + use test_log::test; + + #[test] + fn compression_serialize_none() { + let serialized = CompressionType::Lz4.encode_into_vec(); + assert_eq!(1, serialized.len()); + } + } +} diff --git a/crates/lsm-tree/src/config/block_size.rs b/crates/lsm-tree/src/config/block_size.rs new file mode 100644 index 000000000..e402a6f45 --- /dev/null +++ b/crates/lsm-tree/src/config/block_size.rs @@ -0,0 +1,44 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Block size policy +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct BlockSizePolicy(Vec); + +impl std::ops::Deref for BlockSizePolicy { + type Target = [u32]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl BlockSizePolicy { + pub(crate) fn get(&self, level: usize) -> u32 { + #[expect(clippy::expect_used, reason = "policy is expected not to be empty")] + self.0 + .get(level) + .copied() + .unwrap_or_else(|| self.last().copied().expect("policy should not be empty")) + } + + /// Uses the same block size in every level. + #[must_use] + pub fn all(c: u32) -> Self { + Self(vec![c]) + } + + /// Constructs a custom block size policy. + /// + /// # Panics + /// + /// Panics if the policy is empty or contains more than 255 elements. + #[must_use] + pub fn new(policy: impl Into>) -> Self { + let policy = policy.into(); + assert!(!policy.is_empty(), "compression policy may not be empty"); + assert!(policy.len() <= 255, "compression policy is too large"); + Self(policy) + } +} diff --git a/crates/lsm-tree/src/config/compression.rs b/crates/lsm-tree/src/config/compression.rs new file mode 100644 index 000000000..61dc017e6 --- /dev/null +++ b/crates/lsm-tree/src/config/compression.rs @@ -0,0 +1,52 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::CompressionType; + +/// Compression policy +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CompressionPolicy(Vec); + +impl std::ops::Deref for CompressionPolicy { + type Target = [CompressionType]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl CompressionPolicy { + pub(crate) fn get(&self, level: usize) -> CompressionType { + #[expect(clippy::expect_used, reason = "policy is expected not to be empty")] + self.0 + .get(level) + .copied() + .unwrap_or_else(|| self.last().copied().expect("policy should not be empty")) + } + + /// Disables all compression. + #[must_use] + pub fn disabled() -> Self { + Self::all(CompressionType::None) + } + + /// Uses the same compression in every level. + #[must_use] + pub fn all(c: CompressionType) -> Self { + Self(vec![c]) + } + + /// Constructs a custom compression policy. + /// + /// # Panics + /// + /// Panics if the policy is empty or contains more than 255 elements. + #[must_use] + pub fn new(policy: impl Into>) -> Self { + let policy = policy.into(); + assert!(!policy.is_empty(), "compression policy may not be empty"); + assert!(policy.len() <= 255, "compression policy is too large"); + Self(policy) + } +} diff --git a/crates/lsm-tree/src/config/filter.rs b/crates/lsm-tree/src/config/filter.rs new file mode 100644 index 000000000..0136de725 --- /dev/null +++ b/crates/lsm-tree/src/config/filter.rs @@ -0,0 +1,66 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub use crate::table::filter::BloomConstructionPolicy; + +/// Filter policy entry +/// +/// Each level can be configured with a different filter type and bits per key +#[derive(Copy, Debug, Clone, PartialEq)] +pub enum FilterPolicyEntry { + /// Skip filter construction + None, + + /// Standard bloom filter with K bits per key + Bloom(BloomConstructionPolicy), +} + +/// Filter policy +#[derive(Debug, Clone, PartialEq)] +pub struct FilterPolicy(Vec); + +impl std::ops::Deref for FilterPolicy { + type Target = [FilterPolicyEntry]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl FilterPolicy { + pub(crate) fn get(&self, level: usize) -> FilterPolicyEntry { + #[expect(clippy::expect_used, reason = "policy is expected not to be empty")] + self.0 + .get(level) + .copied() + .unwrap_or_else(|| self.last().copied().expect("policy should not be empty")) + } + + /// Disables all filters. + /// + /// **Not recommended unless you know what you are doing!** + #[must_use] + pub fn disabled() -> Self { + Self::all(FilterPolicyEntry::None) + } + + /// Uses the same block size in every level. + #[must_use] + pub fn all(c: FilterPolicyEntry) -> Self { + Self(vec![c]) + } + + /// Constructs a custom block size policy. + /// + /// # Panics + /// + /// Panics if the policy is empty or contains more than 255 elements. + #[must_use] + pub fn new(policy: impl Into>) -> Self { + let policy = policy.into(); + assert!(!policy.is_empty(), "compression policy may not be empty"); + assert!(policy.len() <= 255, "compression policy is too large"); + Self(policy) + } +} diff --git a/crates/lsm-tree/src/config/hash_ratio.rs b/crates/lsm-tree/src/config/hash_ratio.rs new file mode 100644 index 000000000..b0fd623f1 --- /dev/null +++ b/crates/lsm-tree/src/config/hash_ratio.rs @@ -0,0 +1,44 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Hash ratio policy +#[derive(Debug, Clone, PartialEq)] +pub struct HashRatioPolicy(Vec); + +impl std::ops::Deref for HashRatioPolicy { + type Target = [f32]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl HashRatioPolicy { + pub(crate) fn get(&self, level: usize) -> f32 { + #[expect(clippy::expect_used, reason = "policy is expected not to be empty")] + self.0 + .get(level) + .copied() + .unwrap_or_else(|| self.last().copied().expect("policy should not be empty")) + } + + /// Uses the same block size in every level. + #[must_use] + pub fn all(c: f32) -> Self { + Self(vec![c]) + } + + /// Constructs a custom block size policy. + /// + /// # Panics + /// + /// Panics if the policy is empty or contains more than 255 elements. + #[must_use] + pub fn new(policy: impl Into>) -> Self { + let policy = policy.into(); + assert!(!policy.is_empty(), "compression policy may not be empty"); + assert!(policy.len() <= 255, "compression policy is too large"); + Self(policy) + } +} diff --git a/crates/lsm-tree/src/config/mod.rs b/crates/lsm-tree/src/config/mod.rs new file mode 100644 index 000000000..93c6b64da --- /dev/null +++ b/crates/lsm-tree/src/config/mod.rs @@ -0,0 +1,472 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod block_size; +mod compression; +mod filter; +mod hash_ratio; +mod pinning; +mod restart_interval; + +pub use block_size::BlockSizePolicy; +pub use compression::CompressionPolicy; +pub use filter::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry}; +pub use hash_ratio::HashRatioPolicy; +pub use pinning::PinningPolicy; +pub use restart_interval::RestartIntervalPolicy; + +/// Partitioning policy for indexes and filters +pub type PartitioningPolicy = PinningPolicy; + +use crate::{ + compaction::filter::Factory, path::absolute_path, version::DEFAULT_LEVEL_COUNT, AnyTree, + BlobTree, Cache, CompressionType, DescriptorTable, SequenceNumberCounter, Tree, +}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +/// LSM-tree type +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum TreeType { + /// Standard LSM-tree, see [`Tree`] + Standard, + + /// Key-value separated LSM-tree, see [`BlobTree`] + Blob, +} + +impl From for u8 { + fn from(val: TreeType) -> Self { + match val { + TreeType::Standard => 0, + TreeType::Blob => 1, + } + } +} + +impl TryFrom for TreeType { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Standard), + 1 => Ok(Self::Blob), + _ => Err(()), + } + } +} + +const DEFAULT_FILE_FOLDER: &str = ".lsm.data"; + +/// Options for key-value separation +#[derive(Clone, Debug, PartialEq)] +pub struct KvSeparationOptions { + /// What type of compression is used for blobs + #[doc(hidden)] + pub compression: CompressionType, + + /// Blob file target size in bytes + #[doc(hidden)] + pub file_target_size: u64, + + /// Key-value separation threshold in bytes + #[doc(hidden)] + pub separation_threshold: u32, + + #[doc(hidden)] + pub staleness_threshold: f32, + + #[doc(hidden)] + pub age_cutoff: f32, +} + +impl Default for KvSeparationOptions { + fn default() -> Self { + Self { + #[cfg(feature="lz4")] + compression: CompressionType::Lz4, + + #[cfg(not(feature="lz4"))] + compression: CompressionType::None, + + file_target_size: /* 64 MiB */ 64 * 1_024 * 1_024, + separation_threshold: /* 1 KiB */ 1_024, + + staleness_threshold: 0.25, + age_cutoff: 0.25, + } + } +} + +impl KvSeparationOptions { + /// Sets the blob compression method. + #[must_use] + pub fn compression(mut self, compression: CompressionType) -> Self { + self.compression = compression; + self + } + + /// Sets the target size of blob files. + /// + /// Smaller blob files allow more granular garbage collection + /// which allows lower space amp for lower write I/O cost. + /// + /// Larger blob files decrease the number of files on disk and maintenance + /// overhead. + /// + /// Defaults to 64 MiB. + #[must_use] + pub fn file_target_size(mut self, bytes: u64) -> Self { + self.file_target_size = bytes; + self + } + + /// Sets the key-value separation threshold in bytes. + /// + /// Smaller value will reduce compaction overhead and thus write amplification, + /// at the cost of lower read performance. + /// + /// Defaults to 1 KiB. + #[must_use] + pub fn separation_threshold(mut self, bytes: u32) -> Self { + self.separation_threshold = bytes; + self + } + + /// Sets the staleness threshold percentage. + /// + /// The staleness percentage determines how much a blob file needs to be fragmented to be + /// picked up by the garbage collection. + /// + /// Defaults to 33%. + #[must_use] + pub fn staleness_threshold(mut self, ratio: f32) -> Self { + self.staleness_threshold = ratio; + self + } + + /// Sets the age cutoff threshold. + /// + /// Defaults to 20%. + #[must_use] + pub fn age_cutoff(mut self, ratio: f32) -> Self { + self.age_cutoff = ratio; + self + } +} + +/// Tree configuration builder +pub struct Config { + /// Folder path + #[doc(hidden)] + pub path: PathBuf, + + /// Block cache to use + #[doc(hidden)] + pub cache: Arc, + + /// Descriptor table to use + #[doc(hidden)] + pub descriptor_table: Option>, + + /// Number of levels of the LSM tree (depth of tree) + /// + /// Once set, the level count is fixed (in the "manifest" file) + pub level_count: u8, + + /// What type of compression is used for data blocks + pub data_block_compression_policy: CompressionPolicy, + + /// What type of compression is used for index blocks + pub index_block_compression_policy: CompressionPolicy, + + /// Restart interval inside data blocks + pub data_block_restart_interval_policy: RestartIntervalPolicy, + + /// Restart interval inside index blocks + pub index_block_restart_interval_policy: RestartIntervalPolicy, + + /// Block size of data blocks + pub data_block_size_policy: BlockSizePolicy, + + /// Whether to pin index blocks + pub index_block_pinning_policy: PinningPolicy, + + /// Whether to pin filter blocks + pub filter_block_pinning_policy: PinningPolicy, + + /// Whether to pin top level index of partitioned index + + /// Whether to pin top level index of partitioned filter + + /// Data block hash ratio + pub data_block_hash_ratio_policy: HashRatioPolicy, + + /// Whether to partition index blocks + pub index_block_partitioning_policy: PartitioningPolicy, + + /// Whether to partition filter blocks + pub filter_block_partitioning_policy: PartitioningPolicy, + + /// Partition size when using partitioned indexes + + /// Partition size when using partitioned filters + + /// If `true`, the last level will not build filters, reducing the filter size of a database + /// by ~90% typically + pub(crate) expect_point_read_hits: bool, + + /// Filter construction policy + pub filter_policy: FilterPolicy, + + /// Compaction filter factory + pub compaction_filter_factory: Option>, + + #[doc(hidden)] + pub kv_separation_opts: Option, + + /// The global sequence number generator + /// + /// Should be shared between multple trees of a database + pub(crate) seqno: SequenceNumberCounter, + + pub(crate) visible_seqno: SequenceNumberCounter, +} + +// TODO: remove default? +impl Default for Config { + fn default() -> Self { + Self { + path: absolute_path(Path::new(DEFAULT_FILE_FOLDER)), + descriptor_table: Some(Arc::new(DescriptorTable::new(256))), + seqno: SequenceNumberCounter::default(), + visible_seqno: SequenceNumberCounter::default(), + + cache: Arc::new(Cache::with_capacity_bytes( + /* 16 MiB */ 16 * 1_024 * 1_024, + )), + + data_block_restart_interval_policy: RestartIntervalPolicy::all(16), + index_block_restart_interval_policy: RestartIntervalPolicy::all(1), + + level_count: DEFAULT_LEVEL_COUNT, + + data_block_size_policy: BlockSizePolicy::all(4_096), + + index_block_pinning_policy: PinningPolicy::new([true, true, false]), + filter_block_pinning_policy: PinningPolicy::new([true, false]), + + index_block_partitioning_policy: PinningPolicy::new([false, false, false, true]), + filter_block_partitioning_policy: PinningPolicy::new([false, false, false, true]), + + data_block_compression_policy: ({ + #[cfg(feature = "lz4")] + let c = CompressionPolicy::new([CompressionType::None, CompressionType::Lz4]); + + #[cfg(not(feature = "lz4"))] + let c = CompressionPolicy::new([CompressionType::None]); + + c + }), + index_block_compression_policy: CompressionPolicy::all(CompressionType::None), + + data_block_hash_ratio_policy: HashRatioPolicy::all(0.0), + + filter_policy: FilterPolicy::all(FilterPolicyEntry::Bloom( + BloomConstructionPolicy::BitsPerKey(10.0), + )), + + compaction_filter_factory: None, + + expect_point_read_hits: false, + + kv_separation_opts: None, + } + } +} + +impl Config { + /// Initializes a new config + pub fn new>( + path: P, + seqno: SequenceNumberCounter, + visible_seqno: SequenceNumberCounter, + ) -> Self { + Self { + path: absolute_path(path.as_ref()), + seqno, + visible_seqno, + ..Default::default() + } + } + + /// Sets the global cache. + /// + /// You can create a global [`Cache`] and share it between multiple + /// trees to cap global cache memory usage. + /// + /// Defaults to a cache with 16 MiB of capacity *per tree*. + #[must_use] + pub fn use_cache(mut self, cache: Arc) -> Self { + self.cache = cache; + self + } + + /// Sets the file descriptor cache. + /// + /// Can be shared across trees. + #[must_use] + pub fn use_descriptor_table(mut self, descriptor_table: Option>) -> Self { + self.descriptor_table = descriptor_table; + self + } + + /// If `true`, the last level will not build filters, reducing the filter size of a database + /// by ~90% typically. + /// + /// **Enable this only if you know that point reads generally are expected to find a key-value pair.** + #[must_use] + pub fn expect_point_read_hits(mut self, b: bool) -> Self { + self.expect_point_read_hits = b; + self + } + + /// Sets the partitioning policy for filter blocks. + #[must_use] + pub fn filter_block_partitioning_policy(mut self, policy: PinningPolicy) -> Self { + self.filter_block_partitioning_policy = policy; + self + } + + /// Sets the partitioning policy for index blocks. + #[must_use] + pub fn index_block_partitioning_policy(mut self, policy: PinningPolicy) -> Self { + self.index_block_partitioning_policy = policy; + self + } + + /// Sets the pinning policy for filter blocks. + #[must_use] + pub fn filter_block_pinning_policy(mut self, policy: PinningPolicy) -> Self { + self.filter_block_pinning_policy = policy; + self + } + + /// Sets the pinning policy for index blocks. + #[must_use] + pub fn index_block_pinning_policy(mut self, policy: PinningPolicy) -> Self { + self.index_block_pinning_policy = policy; + self + } + + /// Sets the restart interval inside data blocks. + /// + /// A higher restart interval saves space while increasing lookup times + /// inside data blocks. + /// + /// Default = 16 + #[must_use] + pub fn data_block_restart_interval_policy(mut self, policy: RestartIntervalPolicy) -> Self { + self.data_block_restart_interval_policy = policy; + self + } + + // TODO: not supported yet in index blocks + // /// Sets the restart interval inside index blocks. + // /// + // /// A higher restart interval saves space while increasing lookup times + // /// inside index blocks. + // /// + // /// Default = 1 + // #[must_use] + // pub fn index_block_restart_interval_policy(mut self, policy: RestartIntervalPolicy) -> Self { + // self.index_block_restart_interval_policy = policy; + // self + // } + + /// Sets the filter construction policy. + #[must_use] + pub fn filter_policy(mut self, policy: FilterPolicy) -> Self { + self.filter_policy = policy; + self + } + + /// Sets the compression method for data blocks. + #[must_use] + pub fn data_block_compression_policy(mut self, policy: CompressionPolicy) -> Self { + self.data_block_compression_policy = policy; + self + } + + /// Sets the compression method for index blocks. + #[must_use] + pub fn index_block_compression_policy(mut self, policy: CompressionPolicy) -> Self { + self.index_block_compression_policy = policy; + self + } + + // TODO: level count is fixed to 7 right now + // /// Sets the number of levels of the LSM tree (depth of tree). + // /// + // /// Defaults to 7, like `LevelDB` and `RocksDB`. + // /// + // /// Cannot be changed once set. + // /// + // /// # Panics + // /// + // /// Panics if `n` is 0. + // #[must_use] + // pub fn level_count(mut self, n: u8) -> Self { + // assert!(n > 0); + + // self.level_count = n; + // self + // } + + /// Sets the data block size policy. + #[must_use] + pub fn data_block_size_policy(mut self, policy: BlockSizePolicy) -> Self { + self.data_block_size_policy = policy; + self + } + + /// Sets the hash ratio policy for data blocks. + /// + /// If greater than 0.0, a hash index is embedded into data blocks that can speed up reads + /// inside the data block. + #[must_use] + pub fn data_block_hash_ratio_policy(mut self, policy: HashRatioPolicy) -> Self { + self.data_block_hash_ratio_policy = policy; + self + } + + /// Toggles key-value separation. + #[must_use] + pub fn with_kv_separation(mut self, opts: Option) -> Self { + self.kv_separation_opts = opts; + self + } + + /// Installs a custom compaction filter. + #[must_use] + pub fn with_compaction_filter_factory(mut self, factory: Option>) -> Self { + self.compaction_filter_factory = factory; + self + } + + /// Opens a tree using the config. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn open(self) -> crate::Result { + Ok(if self.kv_separation_opts.is_some() { + AnyTree::Blob(BlobTree::open(self)?) + } else { + AnyTree::Standard(Tree::open(self)?) + }) + } +} diff --git a/crates/lsm-tree/src/config/pinning.rs b/crates/lsm-tree/src/config/pinning.rs new file mode 100644 index 000000000..e5de3fc37 --- /dev/null +++ b/crates/lsm-tree/src/config/pinning.rs @@ -0,0 +1,50 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Pinning policy +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct PinningPolicy(Vec); + +impl std::ops::Deref for PinningPolicy { + type Target = [bool]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PinningPolicy { + pub(crate) fn get(&self, level: usize) -> bool { + #[expect(clippy::expect_used, reason = "policy is expected not to be empty")] + self.0 + .get(level) + .copied() + .unwrap_or_else(|| self.last().copied().expect("policy should not be empty")) + } + + /// Uses the same policy in every level. + #[must_use] + pub fn all(c: bool) -> Self { + Self(vec![c]) + } + + /// Fully disables pinning. + #[must_use] + pub fn disabled() -> Self { + Self::all(false) + } + + /// Constructs a custom policy. + /// + /// # Panics + /// + /// Panics if the policy is empty or contains more than 255 elements. + #[must_use] + pub fn new(policy: impl Into>) -> Self { + let policy = policy.into(); + assert!(!policy.is_empty(), "compression policy may not be empty"); + assert!(policy.len() <= 255, "compression policy is too large"); + Self(policy) + } +} diff --git a/crates/lsm-tree/src/config/restart_interval.rs b/crates/lsm-tree/src/config/restart_interval.rs new file mode 100644 index 000000000..c6523c601 --- /dev/null +++ b/crates/lsm-tree/src/config/restart_interval.rs @@ -0,0 +1,46 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Restart interval policy +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct RestartIntervalPolicy(Vec); + +impl std::ops::Deref for RestartIntervalPolicy { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl RestartIntervalPolicy { + pub(crate) fn get(&self, level: usize) -> u8 { + #[expect(clippy::expect_used, reason = "policy is expected not to be empty")] + self.0 + .get(level) + .copied() + .unwrap_or_else(|| self.last().copied().expect("policy should not be empty")) + } + + // TODO: accept Vec... Into>? or owned + + /// Uses the same block size in every level. + #[must_use] + pub fn all(c: u8) -> Self { + Self(vec![c]) + } + + /// Constructs a custom block size policy. + /// + /// # Panics + /// + /// Panics if the policy is empty or contains more than 255 elements. + #[must_use] + pub fn new(policy: impl Into>) -> Self { + let policy = policy.into(); + assert!(!policy.is_empty(), "compression policy may not be empty"); + assert!(policy.len() <= 255, "compression policy is too large"); + Self(policy) + } +} diff --git a/crates/lsm-tree/src/descriptor_table.rs b/crates/lsm-tree/src/descriptor_table.rs new file mode 100644 index 000000000..12df8db2b --- /dev/null +++ b/crates/lsm-tree/src/descriptor_table.rs @@ -0,0 +1,73 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::GlobalTableId; +use quick_cache::{sync::Cache as QuickCache, UnitWeighter}; +use std::{fs::File, sync::Arc}; + +const TAG_BLOCK: u8 = 0; +const TAG_BLOB: u8 = 1; + +type Item = Arc; + +#[derive(Eq, std::hash::Hash, PartialEq)] +struct CacheKey(u8, u64, u64); + +/// Caches file descriptors to tables and blob files +pub struct DescriptorTable { + inner: QuickCache, +} + +impl DescriptorTable { + #[must_use] + pub fn new(capacity: usize) -> Self { + use quick_cache::sync::DefaultLifecycle; + + let quick_cache = QuickCache::with( + 1_000, + capacity as u64, + UnitWeighter, + rustc_hash::FxBuildHasher, + DefaultLifecycle::default(), + ); + + Self { inner: quick_cache } + } + + pub(crate) fn len(&self) -> usize { + self.inner.len() + } + + #[must_use] + pub fn access_for_table(&self, id: &GlobalTableId) -> Option> { + let key = CacheKey(TAG_BLOCK, id.tree_id(), id.table_id()); + self.inner.get(&key) + } + + pub fn insert_for_table(&self, id: GlobalTableId, item: Item) { + let key = CacheKey(TAG_BLOCK, id.tree_id(), id.table_id()); + self.inner.insert(key, item); + } + + #[must_use] + pub fn access_for_blob_file(&self, id: &GlobalTableId) -> Option> { + let key = CacheKey(TAG_BLOB, id.tree_id(), id.table_id()); + self.inner.get(&key) + } + + pub fn insert_for_blob_file(&self, id: GlobalTableId, item: Item) { + let key = CacheKey(TAG_BLOB, id.tree_id(), id.table_id()); + self.inner.insert(key, item); + } + + pub fn remove_for_table(&self, id: &GlobalTableId) { + let key = CacheKey(TAG_BLOCK, id.tree_id(), id.table_id()); + self.inner.remove(&key); + } + + pub fn remove_for_blob_file(&self, id: &GlobalTableId) { + let key = CacheKey(TAG_BLOB, id.tree_id(), id.table_id()); + self.inner.remove(&key); + } +} diff --git a/crates/lsm-tree/src/double_ended_peekable.rs b/crates/lsm-tree/src/double_ended_peekable.rs new file mode 100644 index 000000000..4897e3cd6 --- /dev/null +++ b/crates/lsm-tree/src/double_ended_peekable.rs @@ -0,0 +1,197 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +//! A fork of +//! to allow accessing the inner type +//! +//! Also changes the generics a bit so it plays well with `self_cell`. + +use core::{fmt::Debug, hash::Hash, hint::unreachable_unchecked, mem}; + +/// An _extension trait_ to create [`DoubleEndedPeekable`]. +/// +/// This has a blanket implementation for all types that implement [`Iterator`]. +pub trait DoubleEndedPeekableExt> { + /// Creates an iterator which works similarly to [`Peekable`], but also provides additional + /// functions if the underlying type implements [`DoubleEndedIterator`]. + /// + /// See [`DoubleEndedPeekable`] for more information. + /// + /// [`Peekable`]: core::iter::Peekable + fn double_ended_peekable(self) -> DoubleEndedPeekable; +} + +impl DoubleEndedPeekableExt for I +where + I: Iterator, +{ + #[inline] + fn double_ended_peekable(self) -> DoubleEndedPeekable { + DoubleEndedPeekable { + iter: self, + front: MaybePeeked::Unpeeked, + back: MaybePeeked::Unpeeked, + } + } +} + +/// An advanced version of [`Peekable`] that works well with double-ended iterators. +/// +/// This `struct` is created by the [`double_ended_peekable`] method on [`DoubleEndedPeekableExt`]. +/// +/// [`Peekable`]: core::iter::Peekable +/// [`double_ended_peekable`]: DoubleEndedPeekableExt::double_ended_peekable +pub struct DoubleEndedPeekable> { + iter: I, + front: MaybePeeked, + back: MaybePeeked, +} + +impl DoubleEndedPeekable +where + I: Iterator, +{ + pub fn inner_mut(&mut self) -> &mut I { + &mut self.iter + } + + /// Returns a reference to the `next()` value without advancing the iterator. + /// + /// See [`Peekable::peek`] for more information. + /// + /// [`Peekable::peek`]: core::iter::Peekable::peek + #[inline] + pub fn peek(&mut self) -> Option<&I::Item> { + self.front + .get_peeked_or_insert_with(|| self.iter.next()) + .as_ref() + .or_else(|| self.back.peeked_value_ref()) + } + + /// Consumes and returns the next value of this iterator if a condition is true. + /// + /// See [`Peekable::next_if`] for more information. + /// + /// [`Peekable::next_if`]: core::iter::Peekable::next_if + #[inline] + pub fn next_if(&mut self, func: impl FnOnce(&I::Item) -> bool) -> Option { + match self.next() { + Some(item) if func(&item) => Some(item), + other => { + debug_assert!(self.front.is_unpeeked()); + self.front = MaybePeeked::Peeked(other); + None + } + } + } +} + +impl DoubleEndedPeekable +where + I: DoubleEndedIterator, +{ + /// Returns a reference to the `next_back()` value without advancing the _back_ of the iterator. + /// + /// Like [`next_back`], if there is a value, it is wrapped in a `Some(T)`. + /// But if the iteration is over, `None` is returned. + /// + /// [`next_back`]: DoubleEndedIterator::next_back + /// + /// Because `peek_back()` returns a reference, and many iterators iterate over references, + /// there can be a possibly confusing situation where the return value is a double reference. + #[inline] + pub fn peek_back(&mut self) -> Option<&I::Item> { + self.back + .get_peeked_or_insert_with(|| self.iter.next_back()) + .as_ref() + .or_else(|| self.front.peeked_value_ref()) + } +} + +impl Iterator for DoubleEndedPeekable +where + I: Iterator, +{ + type Item = I::Item; + + #[inline] + fn next(&mut self) -> Option { + match self.front.take() { + MaybePeeked::Peeked(out @ Some(_)) => out, + MaybePeeked::Peeked(None) => self.back.take().into_peeked_value(), + MaybePeeked::Unpeeked => match self.iter.next() { + item @ Some(_) => item, + None => self.back.take().into_peeked_value(), + }, + } + } +} + +impl DoubleEndedIterator for DoubleEndedPeekable +where + I: DoubleEndedIterator, +{ + #[inline] + fn next_back(&mut self) -> Option { + match self.back.take() { + MaybePeeked::Peeked(out @ Some(_)) => out, + MaybePeeked::Peeked(None) => self.front.take().into_peeked_value(), + MaybePeeked::Unpeeked => match self.iter.next_back() { + out @ Some(_) => out, + None => self.front.take().into_peeked_value(), + }, + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum MaybePeeked { + #[default] + Unpeeked, + Peeked(Option), +} + +impl MaybePeeked { + fn get_peeked_or_insert_with(&mut self, f: F) -> &mut Option + where + F: FnOnce() -> Option, + { + if matches!(self, Self::Unpeeked) { + *self = Self::Peeked(f()); + } + + let Self::Peeked(peeked) = self else { + // SAFETY: it cannot be `Unpeeked` because that case has been just replaced with + // `Peeked`, and we only have two possible states. + #[expect(unsafe_code, reason = "see safety")] + unsafe { + unreachable_unchecked() + } + }; + + peeked + } + + const fn peeked_value_ref(&self) -> Option<&T> { + match self { + Self::Unpeeked | Self::Peeked(None) => None, + Self::Peeked(Some(peeked)) => Some(peeked), + } + } + + fn take(&mut self) -> Self { + mem::replace(self, Self::Unpeeked) + } + + fn into_peeked_value(self) -> Option { + match self { + Self::Unpeeked | Self::Peeked(None) => None, + Self::Peeked(Some(peeked)) => Some(peeked), + } + } + + const fn is_unpeeked(&self) -> bool { + matches!(self, Self::Unpeeked) + } +} diff --git a/crates/lsm-tree/src/error.rs b/crates/lsm-tree/src/error.rs new file mode 100644 index 000000000..530da2a56 --- /dev/null +++ b/crates/lsm-tree/src/error.rs @@ -0,0 +1,94 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{Checksum, CompressionType}; + +/// Represents errors that can occur in the LSM-tree +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// I/O error + Io(std::io::Error), + + /// Decompression failed + Decompress(CompressionType), + + /// Invalid or unparsable data format version + InvalidVersion(u8), + + /// Some required files could not be recovered from disk + Unrecoverable, + + /// Checksum mismatch + ChecksumMismatch { + /// Checksum of loaded block + got: Checksum, + + /// Checksum that was saved in block header + expected: Checksum, + }, + + /// Invalid enum tag + InvalidTag((&'static str, u8)), + + /// Invalid block trailer + InvalidTrailer, + + /// Invalid block header + InvalidHeader(&'static str), + + /// UTF-8 error + Utf8(std::str::Utf8Error), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "LsmTreeError: {self:?}") + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io(e) => Some(e), + _ => None, + } + } +} + +impl From for Error { + fn from(value: sfa::Error) -> Self { + match value { + sfa::Error::Io(e) => Self::from(e), + sfa::Error::ChecksumMismatch { got, expected } => { + log::error!("Archive ToC checksum mismatch"); + Self::ChecksumMismatch { + got: got.into(), + expected: expected.into(), + } + } + sfa::Error::InvalidHeader => { + log::error!("Invalid archive header"); + Self::Unrecoverable + } + sfa::Error::InvalidVersion => { + log::error!("Invalid archive version"); + Self::Unrecoverable + } + sfa::Error::UnsupportedChecksumType => { + log::error!("Invalid archive checksum type"); + Self::Unrecoverable + } + } + } +} + +impl From for Error { + fn from(value: std::io::Error) -> Self { + Self::Io(value) + } +} + +/// Tree result +pub type Result = std::result::Result; diff --git a/crates/lsm-tree/src/file.rs b/crates/lsm-tree/src/file.rs new file mode 100644 index 000000000..2c006ddef --- /dev/null +++ b/crates/lsm-tree/src/file.rs @@ -0,0 +1,232 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::Slice; +use std::{fs::File, io::Write, path::Path}; + +pub const MAGIC_BYTES: [u8; 4] = [b'L', b'S', b'M', 3]; + +pub const TABLES_FOLDER: &str = "tables"; +pub const BLOBS_FOLDER: &str = "blobs"; +pub const CURRENT_VERSION_FILE: &str = "current"; + +/// Reads bytes from a file using `pread`. +pub fn read_exact(file: &File, offset: u64, size: usize) -> std::io::Result { + // SAFETY: This slice builder starts uninitialized, but we know its length + // + // We use read_at/seek_read which give us the number of bytes read + // If that number does not match the slice length, the function errors, + // so the (partially) uninitialized buffer is discarded + // + // Additionally, generally, block loads furthermore do a checksum check which + // would likely catch the buffer being wrong somehow + #[expect(unsafe_code, reason = "see safety")] + let mut builder = unsafe { Slice::builder_unzeroed(size) }; + + { + let bytes_read: usize; + + #[cfg(unix)] + { + use std::os::unix::fs::FileExt; + + bytes_read = file.read_at(&mut builder, offset)?; + } + + #[cfg(windows)] + { + use std::os::windows::fs::FileExt; + + bytes_read = file.seek_read(&mut builder, offset)?; + } + + #[cfg(not(any(unix, windows)))] + { + compile_error!("unsupported platform"); + unimplemented!(); + } + + if bytes_read != size { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("read_exact({bytes_read}) at {offset} did not read enough bytes {size}; file has length {}", file.metadata()?.len()), + )); + } + } + + Ok(builder.freeze().into()) +} + +/// Runs an I/O operation, retrying transient Windows errors with backoff. +/// +/// External programs (antivirus, indexers, backup agents) may briefly hold files open +/// without sharing flags on Windows, failing renames and deletes that would succeed a +/// moment later; POSIX has no such failure mode, so elsewhere the operation runs once. +pub fn retry_transient_io(mut op: impl FnMut() -> std::io::Result) -> std::io::Result { + #[cfg(windows)] + { + const MAX_ATTEMPTS: u32 = 10; + + let mut delay = std::time::Duration::from_millis(1); + + for _ in 1..MAX_ATTEMPTS { + match op() { + Err(e) if is_transient_windows_error(&e) => { + std::thread::sleep(delay); + delay *= 2; + } + result => return result, + } + } + } + + op() +} + +#[cfg(windows)] +fn is_transient_windows_error(e: &std::io::Error) -> bool { + // ERROR_ACCESS_DENIED, ERROR_SHARING_VIOLATION, ERROR_LOCK_VIOLATION, ERROR_USER_MAPPED_FILE + matches!(e.raw_os_error(), Some(5 | 32 | 33 | 1224)) +} + +/// Persists a named temporary file to its final path, replacing any existing file. +pub fn persist_temp_file(temp_file: tempfile::NamedTempFile, path: &Path) -> std::io::Result<()> { + let mut temp_file = Some(temp_file); + + retry_transient_io(|| { + #[expect(clippy::expect_used, reason = "the temp file is put back on failure")] + temp_file + .take() + .expect("temp file should be present") + .persist(path) + .map(|_| ()) + .map_err(|e| { + temp_file = Some(e.file); + e.error + }) + }) +} + +/// Atomically rewrites a file. +pub fn rewrite_atomic(path: &Path, content: &[u8]) -> std::io::Result<()> { + #[expect( + clippy::expect_used, + reason = "every file should have a parent directory" + )] + let folder = path.parent().expect("should have a parent"); + + let mut temp_file = tempfile::NamedTempFile::new_in(folder)?; + temp_file.write_all(content)?; + temp_file.flush()?; + temp_file.as_file_mut().sync_all()?; + persist_temp_file(temp_file, path)?; + + // TODO: not sure why it fails on Windows... + #[cfg(not(target_os = "windows"))] + { + let file = std::fs::File::open(path)?; + file.sync_all()?; + + #[expect( + clippy::expect_used, + reason = "files should always have a parent directory" + )] + let folder = path.parent().expect("should have parent folder"); + fsync_directory(folder)?; + } + + Ok(()) +} + +#[cfg(not(target_os = "windows"))] +pub fn fsync_directory(path: &Path) -> std::io::Result<()> { + let file = std::fs::File::open(path)?; + debug_assert!(file.metadata()?.is_dir()); + file.sync_all() +} + +#[cfg(target_os = "windows")] +pub fn fsync_directory(path: &Path) -> std::io::Result<()> { + // Cannot fsync directory on Windows + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Write; + use test_log::test; + + #[test] + fn atomic_rewrite() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + + let path = dir.path().join("test.txt"); + { + let mut file = File::create(&path)?; + write!(file, "asdasdasdasdasd")?; + } + + rewrite_atomic(&path, b"newcontent")?; + + let content = std::fs::read_to_string(&path)?; + assert_eq!("newcontent", content); + + Ok(()) + } + + #[test] + fn persist_temp_file_replaces_existing() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + + let path = dir.path().join("test.txt"); + std::fs::write(&path, b"old")?; + + let mut temp_file = tempfile::NamedTempFile::new_in(dir.path())?; + write!(temp_file, "new")?; + persist_temp_file(temp_file, &path)?; + + let content = std::fs::read_to_string(&path)?; + assert_eq!("new", content); + + Ok(()) + } + + #[test] + #[cfg(windows)] + fn retry_transient_io_retries_sharing_violation() { + let mut attempts = 0; + + let result = retry_transient_io(|| { + attempts += 1; + + if attempts < 3 { + // ERROR_SHARING_VIOLATION + Err(std::io::Error::from_raw_os_error(32)) + } else { + Ok(42) + } + }); + + assert!(matches!(result, Ok(42))); + assert_eq!(3, attempts); + } + + #[test] + #[cfg(windows)] + fn retry_transient_io_fails_fast_on_other_errors() { + let mut attempts = 0; + + let result: std::io::Result<()> = retry_transient_io(|| { + attempts += 1; + + // ERROR_FILE_NOT_FOUND + Err(std::io::Error::from_raw_os_error(2)) + }); + + assert!(result.is_err()); + assert_eq!(1, attempts); + } +} diff --git a/crates/lsm-tree/src/file_accessor.rs b/crates/lsm-tree/src/file_accessor.rs new file mode 100644 index 000000000..742205fe8 --- /dev/null +++ b/crates/lsm-tree/src/file_accessor.rs @@ -0,0 +1,70 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::descriptor_table::DescriptorTable; +use crate::GlobalTableId; +use std::{fs::File, sync::Arc}; + +/// Allows accessing a file (either cached or pinned) +#[derive(Clone)] +pub enum FileAccessor { + /// Pinned file descriptor + /// + /// This is used in case file descriptor cache is `None` (to skip cache lookups) + File(Arc), + + /// Access to file descriptor cache + DescriptorTable(Arc), +} + +impl FileAccessor { + #[must_use] + pub fn as_descriptor_table(&self) -> Option<&DescriptorTable> { + match self { + Self::DescriptorTable(d) => Some(d), + Self::File(_) => None, + } + } + + #[must_use] + pub fn access_for_table(&self, table_id: &GlobalTableId) -> Option> { + match self { + Self::File(fd) => Some(fd.clone()), + Self::DescriptorTable(descriptor_table) => descriptor_table.access_for_table(table_id), + } + } + + pub fn insert_for_table(&self, table_id: GlobalTableId, fd: Arc) { + if let Self::DescriptorTable(descriptor_table) = self { + descriptor_table.insert_for_table(table_id, fd); + } + } + + #[must_use] + pub fn access_for_blob_file(&self, table_id: &GlobalTableId) -> Option> { + match self { + Self::File(fd) => Some(fd.clone()), + Self::DescriptorTable(descriptor_table) => { + descriptor_table.access_for_blob_file(table_id) + } + } + } + + pub fn insert_for_blob_file(&self, table_id: GlobalTableId, fd: Arc) { + if let Self::DescriptorTable(descriptor_table) = self { + descriptor_table.insert_for_blob_file(table_id, fd); + } + } +} + +impl std::fmt::Debug for FileAccessor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::File(_) => write!(f, "FileAccessor::Pinned"), + Self::DescriptorTable(_) => { + write!(f, "FileAccessor::Cached") + } + } + } +} diff --git a/crates/lsm-tree/src/format_version.rs b/crates/lsm-tree/src/format_version.rs new file mode 100644 index 000000000..37174af40 --- /dev/null +++ b/crates/lsm-tree/src/format_version.rs @@ -0,0 +1,50 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Disk format version +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum FormatVersion { + /// Version for 1.x.x releases + V1 = 1, + + /// Version for 2.x.x releases + V2, + + /// Version for 3.x.x releases + V3, + + /// Fixed-width key/value lengths in table data blocks + V4, +} + +impl std::fmt::Display for FormatVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", u8::from(*self)) + } +} + +impl From for u8 { + fn from(value: FormatVersion) -> Self { + match value { + FormatVersion::V1 => 1, + FormatVersion::V2 => 2, + FormatVersion::V3 => 3, + FormatVersion::V4 => 4, + } + } +} + +impl TryFrom for FormatVersion { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + 1 => Ok(Self::V1), + 2 => Ok(Self::V2), + 3 => Ok(Self::V3), + 4 => Ok(Self::V4), + _ => Err(()), + } + } +} diff --git a/crates/lsm-tree/src/hash.rs b/crates/lsm-tree/src/hash.rs new file mode 100644 index 000000000..5d5d17168 --- /dev/null +++ b/crates/lsm-tree/src/hash.rs @@ -0,0 +1,33 @@ +/// Generates a 64-bit hash using xxh3. +pub fn hash64(bytes: &[u8]) -> u64 { + xxhash_rust::xxh3::xxh3_64(bytes) +} + +/// Generates a 128-bit hash using xxh3. +pub fn hash128(bytes: &[u8]) -> u128 { + xxhash_rust::xxh3::xxh3_128(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn test_hash64() { + assert_eq!(16_959_823_422_411_450_475, hash64(&[0, 0, 0])); + assert_eq!(8_004_557_073_989_523_290, hash64(&[0, 0, 1])); + } + + #[test] + fn test_hash128() { + assert_eq!( + 321_827_061_816_535_117_015_859_907_874_601_773_163, + hash128(&[0, 0, 0]) + ); + assert_eq!( + 154_036_699_985_066_753_773_347_827_765_470_844_762, + hash128(&[0, 0, 1]) + ); + } +} diff --git a/crates/lsm-tree/src/ingestion.rs b/crates/lsm-tree/src/ingestion.rs new file mode 100644 index 000000000..b829690f9 --- /dev/null +++ b/crates/lsm-tree/src/ingestion.rs @@ -0,0 +1,126 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + blob_tree::ingest::BlobIngestion, tree::ingest::Ingestion, AnyTree, UserKey, UserValue, +}; + +/// Unified ingestion builder over `AnyTree` +// Keep zero allocations and direct dispatch; boxing introduces heap indirection and `dyn` adds virtual dispatch. +// Ingestion calls use `&mut self` in tight loops; the active variant is stable and branch prediction makes the match cheap. +// Allowing this lint preserves hot-path performance at the cost of a larger enum size. +#[expect(clippy::large_enum_variant)] +pub enum AnyIngestion<'a> { + /// Ingestion for a standard LSM-tree + Standard(Ingestion<'a>), + + /// Ingestion for a [`BlobTree`] with KV separation + Blob(BlobIngestion<'a>), +} + +impl AnyIngestion<'_> { + /// Writes a key-value pair. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write, V: Into>( + &mut self, + key: K, + value: V, + ) -> crate::Result<()> { + match self { + Self::Standard(i) => i.write(key.into(), value.into()), + Self::Blob(b) => b.write(key.into(), value.into()), + } + } + + #[doc(hidden)] + pub fn write_prevalidated, V: Into>( + &mut self, + key: K, + value: V, + ) -> crate::Result<()> { + match self { + Self::Standard(i) => i.write_prevalidated(key.into(), value.into()), + Self::Blob(b) => b.write(key.into(), value.into()), + } + } + + /// Writes a tombstone for a key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write_tombstone>(&mut self, key: K) -> crate::Result<()> { + match self { + Self::Standard(i) => i.write_tombstone(key.into()), + Self::Blob(b) => b.write_tombstone(key.into()), + } + } + + /// Writes a weak tombstone for a key. + /// + /// # Examples + /// + /// ``` + /// # use lsm_tree::Config; + /// # let folder = tempfile::tempdir()?; + /// # let tree = Config::new(folder, Default::default(), Default::default()).open()?; + /// # + /// let mut ingestion = tree.ingestion()?; + /// ingestion.write("a", "abc")?; + /// ingestion.write_weak_tombstone("b")?; + /// ingestion.finish()?; + /// # + /// # Ok::<(), lsm_tree::Error>(()) + /// ``` + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write_weak_tombstone>(&mut self, key: K) -> crate::Result<()> { + match self { + Self::Standard(i) => i.write_weak_tombstone(key.into()), + Self::Blob(b) => b.write_weak_tombstone(key.into()), + } + } + + #[doc(hidden)] + pub fn write_prevalidated_weak_tombstone>( + &mut self, + key: K, + ) -> crate::Result<()> { + match self { + Self::Standard(i) => i.write_prevalidated_weak_tombstone(key.into()), + Self::Blob(b) => b.write_weak_tombstone(key.into()), + } + } + + /// Finalizes ingestion and registers created tables (and blob files if present). + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn finish(self) -> crate::Result<()> { + match self { + Self::Standard(i) => i.finish(), + Self::Blob(b) => b.finish(), + } + } +} + +impl AnyTree { + /// Starts an ingestion for any tree type (standard or blob). + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn ingestion(&self) -> crate::Result> { + match self { + Self::Standard(t) => Ok(AnyIngestion::Standard(Ingestion::new(t)?)), + Self::Blob(b) => Ok(AnyIngestion::Blob(BlobIngestion::new(b)?)), + } + } +} diff --git a/crates/lsm-tree/src/iter_guard.rs b/crates/lsm-tree/src/iter_guard.rs new file mode 100644 index 000000000..63fb9312e --- /dev/null +++ b/crates/lsm-tree/src/iter_guard.rs @@ -0,0 +1,64 @@ +use crate::{ + blob_tree::Guard as BlobGuard, tree::Guard as StandardGuard, KvPair, UserKey, UserValue, +}; +use enum_dispatch::enum_dispatch; + +/// Guard to access key-value pairs +#[enum_dispatch] +pub trait IterGuard { + /// Accesses the key-value pair if the predicate returns `true`. + /// + /// The predicate receives the key - if returning false, the value + /// may not be loaded if the tree is key-value separated. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn into_inner_if( + self, + pred: impl Fn(&UserKey) -> bool, + ) -> crate::Result<(UserKey, Option)>; + + /// Accesses the key-value pair. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn into_inner(self) -> crate::Result; + + /// Accesses the key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn key(self) -> crate::Result; + + /// Returns the value size. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn size(self) -> crate::Result; + + /// Accesses the value. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + fn value(self) -> crate::Result + where + Self: Sized, + { + self.into_inner().map(|(_, v)| v) + } +} + +/// Generic iterator value +#[enum_dispatch(IterGuard)] +pub enum IterGuardImpl { + /// Iterator value of a standard LSM-tree + Standard(StandardGuard), + + /// Iterator value of a key-value separated tree + Blob(BlobGuard), +} diff --git a/crates/lsm-tree/src/key.rs b/crates/lsm-tree/src/key.rs new file mode 100644 index 000000000..799897bbe --- /dev/null +++ b/crates/lsm-tree/src/key.rs @@ -0,0 +1,107 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{SeqNo, UserKey, ValueType}; + +#[derive(Clone, Eq)] +pub struct InternalKey { + pub user_key: UserKey, + pub seqno: SeqNo, + pub value_type: ValueType, +} + +impl PartialEq for InternalKey { + fn eq(&self, other: &Self) -> bool { + self.user_key == other.user_key && self.seqno == other.seqno + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +impl std::fmt::Debug for InternalKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:?}:{}:{}", + self.user_key, + self.seqno, + match self.value_type { + ValueType::Value => "V", + ValueType::Tombstone => "T", + ValueType::WeakTombstone => "W", + ValueType::Indirection => "Vb", + }, + ) + } +} + +impl InternalKey { + pub fn new>(user_key: K, seqno: SeqNo, value_type: ValueType) -> Self { + let user_key = user_key.into(); + + assert!( + u16::try_from(user_key.len()).is_ok(), + "keys can be 65535 bytes in length", + ); + + Self { + user_key, + seqno, + value_type, + } + } + + pub fn is_tombstone(&self) -> bool { + self.value_type.is_tombstone() + } +} + +impl PartialOrd for InternalKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +// Order by user key ascending, THEN by sequence number descending +// This is one of the most important functions +// Otherwise queries will not match expected behaviour +impl Ord for InternalKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (&self.user_key, other.seqno).cmp(&(&other.user_key, self.seqno)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn key_order_smoke_test() { + use ValueType::Value as V; + + let mut keys = [ + InternalKey::new(b"d", 3, V), + InternalKey::new(b"a", 0, V), + InternalKey::new(b"b", 0, V), + InternalKey::new(b"a", 2, V), + InternalKey::new(b"b", 1, V), + InternalKey::new(b"a", 1, V), + InternalKey::new(b"c", 3, V), + ]; + keys.sort(); + + assert_eq!( + [ + InternalKey::new(b"a", 2, V), + InternalKey::new(b"a", 1, V), + InternalKey::new(b"a", 0, V), + InternalKey::new(b"b", 1, V), + InternalKey::new(b"b", 0, V), + InternalKey::new(b"c", 3, V), + InternalKey::new(b"d", 3, V), + ], + keys, + ); + } +} diff --git a/crates/lsm-tree/src/key_range.rs b/crates/lsm-tree/src/key_range.rs new file mode 100644 index 000000000..dfc4f2d9d --- /dev/null +++ b/crates/lsm-tree/src/key_range.rs @@ -0,0 +1,343 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{Slice, UserKey}; +use std::ops::Bound; + +/// A key range in the format of [min, max] (inclusive on both sides) +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KeyRange(UserKey, UserKey); + +impl KeyRange { + /// Creates a new key range. + #[must_use] + pub fn new((min, max): (UserKey, UserKey)) -> Self { + Self(min, max) + } + + /// Creates an empty key range. + #[must_use] + pub fn empty() -> Self { + Self(Slice::empty(), Slice::empty()) + } + + /// Returns the lower bound. + #[must_use] + pub fn min(&self) -> &UserKey { + &self.0 + } + + /// Returns the upper bound. + #[must_use] + pub fn max(&self) -> &UserKey { + &self.1 + } + + fn as_tuple(&self) -> (&UserKey, &UserKey) { + (self.min(), self.max()) + } + + /// Returns `true` if the list of key ranges is disjoint + #[must_use] + pub fn is_disjoint(ranges: &[&Self]) -> bool { + for (idx, a) in ranges.iter().enumerate() { + for b in ranges.iter().skip(idx + 1) { + if a.overlaps_with_key_range(b) { + return false; + } + } + } + + true + } + + /// Returns `true` if the key falls within this key range. + #[must_use] + pub fn contains_key(&self, key: &[u8]) -> bool { + let (start, end) = self.as_tuple(); + key >= *start && key <= *end + } + + /// Returns `true` if the `other` is fully contained in this range. + #[must_use] + pub fn contains_range(&self, other: &Self) -> bool { + let (start1, end1) = self.as_tuple(); + let (start2, end2) = other.as_tuple(); + start1 <= start2 && end1 >= end2 + } + + /// Returns `true` if the `other` overlaps at least partially with this range. + #[must_use] + pub fn overlaps_with_key_range(&self, other: &Self) -> bool { + let (start1, end1) = self.as_tuple(); + let (start2, end2) = other.as_tuple(); + end1 >= start2 && start1 <= end2 + } + + /// Returns `true` if the ranges overlap partially or fully. + #[must_use] + pub fn overlaps_with_bounds(&self, bounds: &(Bound<&[u8]>, Bound<&[u8]>)) -> bool { + let (lo, hi) = bounds; + let (my_lo, my_hi) = self.as_tuple(); + + if *lo == Bound::Unbounded && *hi == Bound::Unbounded { + return true; + } + + if *hi == Bound::Unbounded { + return match lo { + Bound::Included(key) => key <= my_hi, + Bound::Excluded(key) => key < my_hi, + Bound::Unbounded => unreachable!(), + }; + } + + if *lo == Bound::Unbounded { + return match hi { + Bound::Included(key) => key >= my_lo, + Bound::Excluded(key) => key > my_lo, + Bound::Unbounded => unreachable!(), + }; + } + + let lo_included = match lo { + Bound::Included(key) => key <= my_hi, + Bound::Excluded(key) => key < my_hi, + Bound::Unbounded => unreachable!(), + }; + + let hi_included = match hi { + Bound::Included(key) => key >= my_lo, + Bound::Excluded(key) => key > my_lo, + Bound::Unbounded => unreachable!(), + }; + + lo_included && hi_included + } + + /// Aggregates a key range. + pub fn aggregate<'a>(mut iter: impl Iterator) -> Self { + let Some(first) = iter.next() else { + return Self::empty(); + }; + + let mut min = first.min(); + let mut max = first.max(); + + for other in iter { + let x = other.min(); + if x < min { + min = x; + } + + let x = other.max(); + if x > max { + max = x; + } + } + + Self(min.clone(), max.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + fn int_key_range(a: u64, b: u64) -> KeyRange { + KeyRange::new((a.to_be_bytes().into(), b.to_be_bytes().into())) + } + + fn string_key_range(a: &str, b: &str) -> KeyRange { + KeyRange::new((a.as_bytes().into(), b.as_bytes().into())) + } + + #[test] + fn key_range_aggregate_1() { + let ranges = [ + int_key_range(2, 4), + int_key_range(0, 4), + int_key_range(7, 10), + ]; + let aggregated = KeyRange::aggregate(ranges.iter()); + let (min, max) = aggregated.as_tuple(); + assert_eq!([0, 0, 0, 0, 0, 0, 0, 0], &**min); + assert_eq!([0, 0, 0, 0, 0, 0, 0, 10], &**max); + } + + #[test] + fn key_range_aggregate_2() { + let ranges = [ + int_key_range(6, 7), + int_key_range(0, 2), + int_key_range(0, 10), + ]; + let aggregated = KeyRange::aggregate(ranges.iter()); + let (min, max) = aggregated.as_tuple(); + assert_eq!([0, 0, 0, 0, 0, 0, 0, 0], &**min); + assert_eq!([0, 0, 0, 0, 0, 0, 0, 10], &**max); + } + + mod is_disjoint { + use super::*; + use test_log::test; + + #[test] + fn key_range_number() { + let ranges = [&int_key_range(0, 4), &int_key_range(0, 4)]; + assert!(!KeyRange::is_disjoint(&ranges)); + } + + #[test] + fn key_range_string() { + let ranges = [&string_key_range("a", "d"), &string_key_range("g", "z")]; + assert!(KeyRange::is_disjoint(&ranges)); + } + + #[test] + fn key_range_not_disjoint() { + let ranges = [&string_key_range("a", "f"), &string_key_range("b", "h")]; + assert!(!KeyRange::is_disjoint(&ranges)); + + let ranges = [ + &string_key_range("a", "d"), + &string_key_range("d", "e"), + &string_key_range("f", "z"), + ]; + assert!(!KeyRange::is_disjoint(&ranges)); + } + } + + mod overflap_key_range { + use super::*; + use test_log::test; + + #[test] + fn key_range_overlap() { + let a = string_key_range("a", "f"); + let b = string_key_range("b", "h"); + assert!(a.overlaps_with_key_range(&b)); + } + + #[test] + fn key_range_overlap_edge() { + let a = string_key_range("a", "f"); + let b = string_key_range("f", "t"); + assert!(a.overlaps_with_key_range(&b)); + } + + #[test] + fn key_range_no_overlap() { + let a = string_key_range("a", "f"); + let b = string_key_range("g", "t"); + assert!(!a.overlaps_with_key_range(&b)); + } + } + + mod overlaps_with_bounds { + use super::*; + use std::ops::Bound::{Excluded, Included, Unbounded}; + use test_log::test; + + #[test] + fn inclusive() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Included(b"key1" as &[u8]), Included(b"key5" as &[u8])); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn exclusive() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Excluded(b"key0" as &[u8]), Excluded(b"key6" as &[u8])); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn no_overlap() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Excluded(b"key5" as &[u8]), Excluded(b"key6" as &[u8])); + assert!(!key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn unbounded() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Unbounded, Unbounded); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_0() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Unbounded, Excluded(b"key1" as &[u8])); + assert!(!key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_1() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Excluded(b"key5" as &[u8]), Unbounded); + assert!(!key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_2() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Unbounded, Included(b"key1" as &[u8])); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_3() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Included(b"key5" as &[u8]), Unbounded); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_4() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Unbounded, Included(b"key5" as &[u8])); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_5() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Unbounded, Included(b"key6" as &[u8])); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_6() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + let bounds = (Included(b"key0" as &[u8]), Unbounded); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + + #[test] + fn semi_open_7() { + let key_range = KeyRange(UserKey::from("key5"), UserKey::from("key8")); + let bounds = (Unbounded, Excluded(b"key6" as &[u8])); + assert!(key_range.overlaps_with_bounds(&bounds)); + } + } + + #[test] + fn key_range_contains_key() { + let key_range = KeyRange(UserKey::from("key1"), UserKey::from("key5")); + assert!(!key_range.contains_key(b"key0")); + assert!(!key_range.contains_key(b"key01")); + assert!(key_range.contains_key(b"key1")); + assert!(key_range.contains_key(b"key2")); + assert!(key_range.contains_key(b"key3")); + assert!(key_range.contains_key(b"key4")); + assert!(key_range.contains_key(b"key4x")); + assert!(key_range.contains_key(b"key5")); + assert!(!key_range.contains_key(b"key5x")); + assert!(!key_range.contains_key(b"key6")); + } +} diff --git a/crates/lsm-tree/src/lib.rs b/crates/lsm-tree/src/lib.rs new file mode 100644 index 000000000..d429d71c5 --- /dev/null +++ b/crates/lsm-tree/src/lib.rs @@ -0,0 +1,207 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +//! A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs). +//! +//! ##### NOTE +//! +//! > This crate only provides a primitive LSM-tree, not a full storage engine. +//! > You probably want to use instead. +//! > For example, it does not ship with a write-ahead log, so writes are not +//! > persisted until manually flushing the memtable. +//! +//! ##### About +//! +//! This crate exports a `Tree` that supports a subset of the `BTreeMap` API. +//! +//! LSM-trees are an alternative to B-trees to persist a sorted list of items (e.g. a database table) +//! on disk and perform fast lookup queries. +//! Instead of updating a disk-based data structure in-place, +//! deltas (inserts and deletes) are added into an in-memory write buffer (`Memtable`). +//! Data is then flushed to disk-resident table files when the write buffer reaches some threshold. +//! +//! Amassing many tables on disk will degrade read performance and waste disk space, so tables +//! can be periodically merged into larger tables in a process called `Compaction`. +//! Different compaction strategies have different advantages and drawbacks, and should be chosen based +//! on the workload characteristics. +//! +//! Because maintaining an efficient structure is deferred to the compaction process, writing to an LSMT +//! is very fast (_O(1)_ complexity). +//! +//! Keys are limited to 65536 bytes, values are limited to 2^32 bytes. As is normal with any kind of storage +//! engine, larger keys and values have a bigger performance impact. + +#![doc(html_logo_url = "https://raw.githubusercontent.com/fjall-rs/lsm-tree/main/logo.png")] +#![doc(html_favicon_url = "https://raw.githubusercontent.com/fjall-rs/lsm-tree/main/logo.png")] +#![deny(clippy::all, missing_docs, clippy::cargo)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::indexing_slicing)] +#![warn(clippy::pedantic, clippy::nursery)] +#![warn(clippy::expect_used)] +#![allow(clippy::missing_const_for_fn)] +#![warn(clippy::multiple_crate_versions)] +#![allow(clippy::option_if_let_else)] +#![warn(clippy::redundant_feature_names)] +#![cfg_attr(coverage_nightly, feature(coverage_attribute))] + +#[doc(hidden)] +pub type HashMap = std::collections::HashMap; + +pub(crate) type HashSet = std::collections::HashSet; + +macro_rules! fail_iter { + ($e:expr) => { + match $e { + Ok(v) => v, + Err(e) => return Some(Err(e.into())), + } + }; +} + +macro_rules! unwrap { + ($x:expr) => {{ + $x.expect("should read") + }}; +} + +mod any_tree; + +mod abstract_tree; + +#[doc(hidden)] +pub mod blob_tree; + +#[doc(hidden)] +mod cache; + +#[doc(hidden)] +pub mod checksum; + +#[doc(hidden)] +pub mod coding; + +pub mod compaction; +mod compression; + +/// Configuration +pub mod config; + +#[doc(hidden)] +pub mod descriptor_table; + +#[doc(hidden)] +pub mod file_accessor; + +mod double_ended_peekable; +mod error; + +#[doc(hidden)] +pub mod file; + +mod hash; +mod ingestion; +mod iter_guard; +mod key; +mod key_range; +mod manifest; +mod memtable; +mod run_reader; +mod run_scanner; + +#[doc(hidden)] +pub mod merge; + +#[cfg(feature = "metrics")] +pub(crate) mod metrics; + +// mod multi_reader; + +#[doc(hidden)] +pub mod mvcc_stream; + +mod path; + +#[doc(hidden)] +pub mod range; + +#[doc(hidden)] +pub mod table; + +mod seqno; +mod slice; +mod slice_windows; + +#[doc(hidden)] +pub mod stop_signal; + +mod format_version; +mod time; +mod tree; + +/// Utility functions +pub mod util; + +mod value; +mod value_type; +mod version; +mod vlog; + +/// User defined key (byte array) +pub type UserKey = Slice; + +/// User defined data (byte array) +pub type UserValue = Slice; + +/// KV-tuple (key + value) +pub type KvPair = (UserKey, UserValue); + +#[doc(hidden)] +pub use { + blob_tree::{handle::BlobIndirection, Guard as BlobGuard}, + checksum::Checksum, + iter_guard::IterGuardImpl, + key_range::KeyRange, + merge::BoxedIterator, + slice::Builder, + table::{GlobalTableId, Table, TableId}, + tree::inner::TreeId, + tree::Guard as StandardGuard, + value::InternalValue, +}; + +pub use { + abstract_tree::AbstractTree, + any_tree::AnyTree, + blob_tree::BlobTree, + cache::Cache, + compression::CompressionType, + config::{Config, KvSeparationOptions, TreeType}, + descriptor_table::DescriptorTable, + error::{Error, Result}, + format_version::FormatVersion, + ingestion::AnyIngestion, + iter_guard::IterGuard as Guard, + memtable::{Memtable, MemtableId}, + seqno::SequenceNumberCounter, + slice::Slice, + tree::Tree, + value::SeqNo, + value_type::ValueType, + vlog::BlobFile, +}; + +#[cfg(feature = "metrics")] +pub use metrics::Metrics; + +#[doc(hidden)] +#[must_use] +#[allow(missing_docs, clippy::missing_errors_doc, clippy::unwrap_used)] +pub fn get_tmp_folder() -> tempfile::TempDir { + if let Ok(p) = std::env::var("LSMT_TMP_FOLDER") { + tempfile::tempdir_in(p) + } else { + tempfile::tempdir() + } + .unwrap() +} diff --git a/crates/lsm-tree/src/manifest.rs b/crates/lsm-tree/src/manifest.rs new file mode 100644 index 000000000..84e70431b --- /dev/null +++ b/crates/lsm-tree/src/manifest.rs @@ -0,0 +1,93 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{checksum::ChecksumType, FormatVersion, TreeType}; +use byteorder::ReadBytesExt; +use std::{io::Read, path::Path}; + +pub struct Manifest { + pub version: FormatVersion, + pub level_count: u8, +} + +impl Manifest { + pub fn decode_from(path: &Path, reader: &sfa::Reader) -> Result { + let toc = reader.toc(); + + let version = { + #[expect( + clippy::expect_used, + reason = "format_version section must exist in manifest" + )] + let section = toc + .section(b"format_version") + .expect("format_version section should exist in manifest"); + + let mut reader = section.buf_reader(path)?; + let version = reader.read_u8()?; + FormatVersion::try_from(version).map_err(|()| crate::Error::InvalidVersion(version))? + }; + + let _tree_type: TreeType = { + #[expect( + clippy::expect_used, + reason = "tree_type section must exist in manifest" + )] + let section = toc + .section(b"tree_type") + .expect("tree_type section should exist in manifest"); + + let mut reader = section.buf_reader(path)?; + let tree_type = reader.read_u8()?; + tree_type + .try_into() + .map_err(|()| crate::Error::InvalidTag(("TreeType", tree_type)))? + }; + + let level_count = { + #[expect( + clippy::expect_used, + reason = "level_count section must exist in manifest" + )] + let section = toc + .section(b"level_count") + .expect("level_count section should exist in manifest"); + + let mut reader = section.buf_reader(path)?; + reader.read_u8()? + }; + + // Currently level count is hard coded to 7 + assert_eq!(7, level_count, "level count should be 7"); + + { + let filter_hash_type = { + #[expect( + clippy::expect_used, + reason = "filter_hash_type section must exist in manifest" + )] + let section = toc + .section(b"filter_hash_type") + .expect("filter_hash_type section should exist in manifest"); + + section + .buf_reader(path)? + .bytes() + .collect::, _>>()? + }; + + // Only one supported right now (and probably forever) + assert_eq!( + &[u8::from(ChecksumType::Xxh3)], + &*filter_hash_type, + "filter_hash_type should be XXH3" + ); + } + + Ok(Self { + version, + level_count, + }) + } +} diff --git a/crates/lsm-tree/src/memtable/mod.rs b/crates/lsm-tree/src/memtable/mod.rs new file mode 100644 index 000000000..ced9c12b7 --- /dev/null +++ b/crates/lsm-tree/src/memtable/mod.rs @@ -0,0 +1,381 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::key::InternalKey; +use crate::{ + value::{InternalValue, SeqNo, UserValue}, + ValueType, +}; +use crossbeam_skiplist::SkipMap; +use std::ops::RangeBounds; +use std::sync::atomic::{AtomicBool, AtomicU64}; + +pub use crate::tree::inner::MemtableId; + +/// The memtable serves as an intermediary, ephemeral, sorted storage for new items +/// +/// When the Memtable exceeds some size, it should be flushed to a table. +pub struct Memtable { + #[doc(hidden)] + pub id: MemtableId, + + /// The actual content, stored in a lock-free skiplist. + #[doc(hidden)] + pub items: SkipMap, + + /// Approximate active memtable size. + /// + /// If this grows too large, a flush is triggered. + pub(crate) approximate_size: AtomicU64, + + /// Highest encountered sequence number. + /// + /// This is used so that `get_highest_seqno` has O(1) complexity. + pub(crate) highest_seqno: AtomicU64, + + pub(crate) requested_rotation: AtomicBool, +} + +impl Memtable { + /// Returns the memtable ID. + pub fn id(&self) -> MemtableId { + self.id + } + + /// Returns `true` if the memtable was already flagged for rotation. + pub fn is_flagged_for_rotation(&self) -> bool { + self.requested_rotation + .load(std::sync::atomic::Ordering::Relaxed) + } + + /// Flags the memtable as requested for rotation. + pub fn flag_rotated(&self) { + self.requested_rotation + .store(true, std::sync::atomic::Ordering::Relaxed); + } + + #[doc(hidden)] + #[must_use] + pub fn new(id: MemtableId) -> Self { + Self { + id, + items: SkipMap::default(), + approximate_size: AtomicU64::default(), + highest_seqno: AtomicU64::default(), + requested_rotation: AtomicBool::default(), + } + } + + /// Creates an iterator over all items. + pub fn iter(&self) -> impl DoubleEndedIterator + '_ { + self.items.iter().map(|entry| InternalValue { + key: entry.key().clone(), + value: entry.value().clone(), + }) + } + + /// Creates an iterator over a range of items. + pub(crate) fn range<'a, R: RangeBounds + 'a>( + &'a self, + range: R, + ) -> impl DoubleEndedIterator + 'a { + self.items.range(range).map(|entry| InternalValue { + key: entry.key().clone(), + value: entry.value().clone(), + }) + } + + /// Returns the item by key if it exists. + /// + /// The item with the highest seqno will be returned, if `seqno` is None. + #[doc(hidden)] + pub fn get(&self, key: &[u8], seqno: SeqNo) -> Option { + if seqno == 0 { + return None; + } + + // NOTE: This range start deserves some explanation... + // InternalKeys are multi-sorted by 2 categories: user_key and Reverse(seqno). (tombstone doesn't really matter) + // We search for the lowest entry that is greater or equal the user's prefix key + // and has the seqno (or lower) we want (because the seqno is stored in reverse order) + // + // Example: We search for "abc" + // + // key -> seqno + // + // a -> 7 + // abc -> 5 <<< This is the lowest key (highest seqno) that matches the key with seqno=None + // abc -> 4 + // abc -> 3 <<< If searching for abc and seqno=4, we would get this + // abcdef -> 6 + // abcdef -> 5 + // + let lower_bound = InternalKey::new(key, seqno - 1, ValueType::Value); + + let mut iter = self + .items + .range(lower_bound..) + .take_while(|entry| &*entry.key().user_key == key); + + iter.next().map(|entry| InternalValue { + key: entry.key().clone(), + value: entry.value().clone(), + }) + } + + /// Gets approximate size of memtable in bytes. + pub fn size(&self) -> u64 { + self.approximate_size + .load(std::sync::atomic::Ordering::Acquire) + } + + /// Counts the number of items in the memtable. + pub fn len(&self) -> usize { + self.items.len() + } + + /// Returns `true` if the memtable is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + /// Inserts an item into the memtable + #[doc(hidden)] + pub fn insert(&self, item: InternalValue) -> (u64, u64) { + #[expect( + clippy::expect_used, + reason = "keys are limited to 16-bit length + values are limited to 32-bit length" + )] + let item_size = + (item.key.user_key.len() + item.value.len() + std::mem::size_of::()) + .try_into() + .expect("should fit into u64"); + + let size_before = self + .approximate_size + .fetch_add(item_size, std::sync::atomic::Ordering::AcqRel); + + let key = InternalKey::new(item.key.user_key, item.key.seqno, item.key.value_type); + self.items.insert(key, item.value); + + self.highest_seqno + .fetch_max(item.key.seqno, std::sync::atomic::Ordering::AcqRel); + + (item_size, size_before + item_size) + } + + /// Returns the highest sequence number in the memtable. + pub fn get_highest_seqno(&self) -> Option { + if self.is_empty() { + None + } else { + Some( + self.highest_seqno + .load(std::sync::atomic::Ordering::Acquire), + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ValueType; + use test_log::test; + + #[test] + #[expect(clippy::unwrap_used)] + fn memtable_mvcc_point_read() { + let memtable = Memtable::new(0); + + memtable.insert(InternalValue::from_components( + *b"hello-key-999991", + *b"hello-value-999991", + 0, + ValueType::Value, + )); + + let item = memtable.get(b"hello-key-99999", SeqNo::MAX); + assert_eq!(None, item); + + let item = memtable.get(b"hello-key-999991", SeqNo::MAX); + assert_eq!(*b"hello-value-999991", &*item.unwrap().value); + + memtable.insert(InternalValue::from_components( + *b"hello-key-999991", + *b"hello-value-999991-2", + 1, + ValueType::Value, + )); + + let item = memtable.get(b"hello-key-99999", SeqNo::MAX); + assert_eq!(None, item); + + let item = memtable.get(b"hello-key-999991", SeqNo::MAX); + assert_eq!((*b"hello-value-999991-2"), &*item.unwrap().value); + + let item = memtable.get(b"hello-key-99999", 1); + assert_eq!(None, item); + + let item = memtable.get(b"hello-key-999991", 1); + assert_eq!((*b"hello-value-999991"), &*item.unwrap().value); + + let item = memtable.get(b"hello-key-99999", 2); + assert_eq!(None, item); + + let item = memtable.get(b"hello-key-999991", 2); + assert_eq!((*b"hello-value-999991-2"), &*item.unwrap().value); + } + + #[test] + fn memtable_get() { + let memtable = Memtable::new(0); + + let value = + InternalValue::from_components(b"abc".to_vec(), b"abc".to_vec(), 0, ValueType::Value); + + memtable.insert(value.clone()); + + assert_eq!(Some(value), memtable.get(b"abc", SeqNo::MAX)); + } + + #[test] + fn memtable_get_highest_seqno() { + let memtable = Memtable::new(0); + + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 0, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 1, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 2, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 3, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 4, + ValueType::Value, + )); + + assert_eq!( + Some(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 4, + ValueType::Value, + )), + memtable.get(b"abc", SeqNo::MAX) + ); + } + + #[test] + fn memtable_get_prefix() { + let memtable = Memtable::new(0); + + memtable.insert(InternalValue::from_components( + b"abc0".to_vec(), + b"abc".to_vec(), + 0, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 255, + ValueType::Value, + )); + + assert_eq!( + Some(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 255, + ValueType::Value, + )), + memtable.get(b"abc", SeqNo::MAX) + ); + + assert_eq!( + Some(InternalValue::from_components( + b"abc0".to_vec(), + b"abc".to_vec(), + 0, + ValueType::Value, + )), + memtable.get(b"abc0", SeqNo::MAX) + ); + } + + #[test] + fn memtable_get_old_version() { + let memtable = Memtable::new(0); + + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 0, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 99, + ValueType::Value, + )); + memtable.insert(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 255, + ValueType::Value, + )); + + assert_eq!( + Some(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 255, + ValueType::Value, + )), + memtable.get(b"abc", SeqNo::MAX) + ); + + assert_eq!( + Some(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 99, + ValueType::Value, + )), + memtable.get(b"abc", 100) + ); + + assert_eq!( + Some(InternalValue::from_components( + b"abc".to_vec(), + b"abc".to_vec(), + 0, + ValueType::Value, + )), + memtable.get(b"abc", 50) + ); + } +} diff --git a/crates/lsm-tree/src/merge.rs b/crates/lsm-tree/src/merge.rs new file mode 100644 index 000000000..a3d1a4116 --- /dev/null +++ b/crates/lsm-tree/src/merge.rs @@ -0,0 +1,176 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::InternalValue; +use interval_heap::IntervalHeap as Heap; + +type IterItem = crate::Result; + +pub type BoxedIterator<'a> = Box + Send + 'a>; + +struct HeapItem(usize, InternalValue); + +impl Eq for HeapItem {} + +impl PartialEq for HeapItem { + fn eq(&self, other: &Self) -> bool { + self.1.key == other.1.key + } +} + +impl Ord for HeapItem { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.1.key.cmp(&other.1.key) + } +} + +impl PartialOrd for HeapItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Merges multiple KV iterators +pub struct Merger { + iterators: Vec, + heap: Heap, + initialized_lo: bool, + initialized_hi: bool, +} + +impl> Merger { + #[must_use] + pub fn new(iterators: Vec) -> Self { + let heap = Heap::with_capacity(iterators.len()); + + let iterators = iterators.into_iter().collect::>(); + + Self { + iterators, + heap, + initialized_lo: false, + initialized_hi: false, + } + } + + fn initialize_lo(&mut self) -> crate::Result<()> { + for (idx, it) in self.iterators.iter_mut().enumerate() { + if let Some(item) = it.next() { + let item = item?; + self.heap.push(HeapItem(idx, item)); + } + } + self.initialized_lo = true; + Ok(()) + } +} + +impl> Merger { + fn initialize_hi(&mut self) -> crate::Result<()> { + for (idx, it) in self.iterators.iter_mut().enumerate() { + if let Some(item) = it.next_back() { + let item = item?; + self.heap.push(HeapItem(idx, item)); + } + } + self.initialized_hi = true; + Ok(()) + } +} + +impl> Iterator for Merger { + type Item = IterItem; + + fn next(&mut self) -> Option { + if !self.initialized_lo { + fail_iter!(self.initialize_lo()); + } + + let min_item = self.heap.pop_min()?; + + #[expect(clippy::indexing_slicing, reason = "we trust the HeapItem index")] + if let Some(next_item) = self.iterators[min_item.0].next() { + let next_item = fail_iter!(next_item); + self.heap.push(HeapItem(min_item.0, next_item)); + } + + Some(Ok(min_item.1)) + } +} + +impl> DoubleEndedIterator for Merger { + fn next_back(&mut self) -> Option { + if !self.initialized_hi { + fail_iter!(self.initialize_hi()); + } + + let max_item = self.heap.pop_max()?; + + #[expect(clippy::indexing_slicing, reason = "we trust the HeapItem index")] + if let Some(next_item) = self.iterators[max_item.0].next_back() { + let next_item = fail_iter!(next_item); + self.heap.push(HeapItem(max_item.0, next_item)); + } + + Some(Ok(max_item.1)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ValueType::Value; + use test_log::test; + + #[test] + #[expect(clippy::unwrap_used)] + fn merge_simple() -> crate::Result<()> { + #[rustfmt::skip] + let a = vec![ + Ok(InternalValue::from_components("a", b"", 0, Value)), + ]; + #[rustfmt::skip] + let b = vec![ + Ok(InternalValue::from_components("b", b"", 0, Value)), + ]; + + let mut iter = Merger::new(vec![a.into_iter(), b.into_iter()]); + + assert_eq!( + iter.next().unwrap()?, + InternalValue::from_components("a", b"", 0, Value), + ); + assert_eq!( + iter.next().unwrap()?, + InternalValue::from_components("b", b"", 0, Value), + ); + assert!(iter.next().is_none(), "iter should be closed"); + + Ok(()) + } + + #[test] + #[ignore = "maybe not needed"] + #[expect(clippy::unwrap_used)] + fn merge_dup() -> crate::Result<()> { + #[rustfmt::skip] + let a = vec![ + Ok(InternalValue::from_components("a", b"", 0, Value)), + ]; + #[rustfmt::skip] + let b = vec![ + Ok(InternalValue::from_components("a", b"", 0, Value)), + ]; + + let mut iter = Merger::new(vec![a.into_iter(), b.into_iter()]); + + assert_eq!( + iter.next().unwrap()?, + InternalValue::from_components("a", b"", 0, Value), + ); + assert!(iter.next().is_none(), "iter should be closed"); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/metrics.rs b/crates/lsm-tree/src/metrics.rs new file mode 100644 index 000000000..fac3baa3e --- /dev/null +++ b/crates/lsm-tree/src/metrics.rs @@ -0,0 +1,212 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::sync::atomic::Ordering::Relaxed; +use std::sync::atomic::{AtomicU64, AtomicUsize}; + +/// Runtime metrics +/// +/// Are not stored durably, so metrics will reset after a restart/crash. +#[derive(Debug, Default)] +pub struct Metrics { + /// Number of times a table file was opened using `fopen()` + pub(crate) table_file_opened_uncached: AtomicUsize, + + /// Number of times a table file was retrieved from descriptor cache + pub(crate) table_file_opened_cached: AtomicUsize, + + /// Number of index blocks that were actually read from disk + pub(crate) index_block_load_io: AtomicUsize, + + /// Number of filter blocks that were actually read from disk + pub(crate) filter_block_load_io: AtomicUsize, + + /// Number of blocks that were actually read from disk + pub(crate) data_block_load_io: AtomicUsize, + + /// Number of blocks that were read from block cache + pub(crate) index_block_load_cached: AtomicUsize, + + /// Number of blocks that were read from block cache + pub(crate) filter_block_load_cached: AtomicUsize, + + /// Number of blocks that were read from block cache + pub(crate) data_block_load_cached: AtomicUsize, + + /// Number of filter queries that were performed + pub(crate) filter_queries: AtomicUsize, + + /// Number of IOs that were skipped due to filter + pub(crate) io_skipped_by_filter: AtomicUsize, + + /// Number of data block bytes that were requested from OS or disk + pub(crate) data_block_io_requested: AtomicU64, + + /// Number of index block bytes that were requested from OS or disk + pub(crate) index_block_io_requested: AtomicU64, + + /// Number of filter block bytes that were requested from OS or disk + pub(crate) filter_block_io_requested: AtomicU64, +} + +#[expect( + clippy::cast_precision_loss, + reason = "metrics can accept precision loss" +)] +impl Metrics { + /// Returns the cache hit rate for file descriptors in percent (0.0 - 1.0). + pub fn table_file_cache_hit_rate(&self) -> f64 { + let uncached = self.table_file_opened_uncached.load(Relaxed) as f64; + let cached = self.table_file_opened_cached.load(Relaxed) as f64; + + if cached + uncached == 0.0 { + 1.0 + } else { + cached / (cached + uncached) + } + } + + /// Number of I/O data block bytes transferred from disk or OS page cache. + pub fn data_block_io(&self) -> u64 { + self.data_block_io_requested.load(Relaxed) + } + + /// Number of I/O index block bytes transferred from disk or OS page cache. + pub fn index_block_io(&self) -> u64 { + self.index_block_io_requested.load(Relaxed) + } + + /// Number of I/O filter block bytes transferred from disk or OS page cache. + pub fn filter_block_io(&self) -> u64 { + self.filter_block_io_requested.load(Relaxed) + } + + /// Number of I/O block bytes transferred from disk or OS page cache. + pub fn block_io(&self) -> u64 { + self.data_block_io_requested.load(Relaxed) + + self.index_block_io_requested.load(Relaxed) + + self.filter_block_io_requested.load(Relaxed) + } + + /// Number of data blocks that were accessed. + pub fn data_block_load_count(&self) -> usize { + self.data_block_load_cached.load(Relaxed) + self.data_block_load_io.load(Relaxed) + } + + /// Number of index blocks that were accessed. + pub fn index_block_load_count(&self) -> usize { + self.index_block_load_cached.load(Relaxed) + self.index_block_load_io.load(Relaxed) + } + + /// Number of filter blocks that were accessed. + pub fn filter_block_load_count(&self) -> usize { + self.filter_block_load_cached.load(Relaxed) + self.filter_block_load_io.load(Relaxed) + } + + /// Number of blocks that were loaded from disk or OS page cache. + pub fn block_load_io_count(&self) -> usize { + self.data_block_load_io.load(Relaxed) + + self.index_block_load_io.load(Relaxed) + + self.filter_block_load_io.load(Relaxed) + } + + /// Number of data blocks that were loaded from disk or OS page cache. + pub fn data_block_load_cached_count(&self) -> usize { + self.data_block_load_cached.load(Relaxed) + } + + /// Number of index blocks that were loaded from disk or OS page cache. + pub fn index_block_load_cached_count(&self) -> usize { + self.index_block_load_cached.load(Relaxed) + } + + /// Number of filter blocks that were loaded from disk or OS page cache. + pub fn filter_block_load_cached_count(&self) -> usize { + self.filter_block_load_cached.load(Relaxed) + } + + /// Number of blocks that were loaded from disk or OS page cache. + pub fn block_load_cached_count(&self) -> usize { + self.data_block_load_cached.load(Relaxed) + + self.index_block_load_cached.load(Relaxed) + + self.filter_block_load_cached.load(Relaxed) + } + + /// Number of blocks that were accessed. + pub fn block_loads(&self) -> usize { + self.block_load_io_count() + self.block_load_cached_count() + } + + /// Data block cache efficiency in percent (0.0 - 1.0). + pub fn data_block_cache_hit_rate(&self) -> f64 { + let queries = self.data_block_load_count() as f64; + let hits = self.data_block_load_cached_count() as f64; + + if queries == 0.0 { + 1.0 + } else { + hits / queries + } + } + + /// Filter block cache efficiency in percent (0.0 - 1.0). + pub fn filter_block_cache_hit_rate(&self) -> f64 { + let queries = self.filter_block_load_count() as f64; + let hits = self.filter_block_load_cached_count() as f64; + + if queries == 0.0 { + 1.0 + } else { + hits / queries + } + } + + /// Index block cache efficiency in percent (0.0 - 1.0). + pub fn index_block_cache_hit_rate(&self) -> f64 { + let queries = self.index_block_load_count() as f64; + let hits = self.index_block_load_cached_count() as f64; + + if queries == 0.0 { + 1.0 + } else { + hits / queries + } + } + + /// Block cache efficiency in percent (0.0 - 1.0). + pub fn block_cache_hit_rate(&self) -> f64 { + let queries = self.block_loads() as f64; + let hits = self.block_load_cached_count() as f64; + + if queries == 0.0 { + 1.0 + } else { + hits / queries + } + } + + /// Filter efficiency in percent (0.0 - 1.0). + /// + /// Represents the ratio of I/O operations avoided due to filter. + pub fn filter_efficiency(&self) -> f64 { + let queries = self.filter_queries.load(Relaxed) as f64; + let io_skipped = self.io_skipped_by_filter.load(Relaxed) as f64; + + if queries == 0.0 { + 1.0 + } else { + io_skipped / queries + } + } + + /// Number of filter queries performed. + pub fn filter_queries(&self) -> usize { + self.filter_queries.load(Relaxed) + } + + /// Number of I/O operations skipped by filter. + pub fn io_skipped_by_filter(&self) -> usize { + self.io_skipped_by_filter.load(Relaxed) + } +} diff --git a/crates/lsm-tree/src/mvcc_stream.rs b/crates/lsm-tree/src/mvcc_stream.rs new file mode 100644 index 000000000..e23246e35 --- /dev/null +++ b/crates/lsm-tree/src/mvcc_stream.rs @@ -0,0 +1,688 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::double_ended_peekable::{DoubleEndedPeekable, DoubleEndedPeekableExt}; +use crate::{InternalValue, UserKey}; + +/// Consumes a stream of KVs and emits a new stream according to MVCC and tombstone rules +/// +/// This iterator is used for read operations. +pub struct MvccStream>> { + inner: DoubleEndedPeekable, I>, +} + +impl>> MvccStream { + /// Initializes a new multi-version-aware iterator. + #[must_use] + pub fn new(iter: I) -> Self { + Self { + inner: iter.double_ended_peekable(), + } + } + + // Drains all entries for the given user key from the front of the iterator. + fn drain_key_min(&mut self, key: &UserKey) -> crate::Result<()> { + loop { + let Some(next) = self.inner.next_if(|kv| { + if let Ok(kv) = kv { + kv.key.user_key == key + } else { + true + } + }) else { + return Ok(()); + }; + + next?; + } + } +} + +impl>> Iterator for MvccStream { + type Item = crate::Result; + + fn next(&mut self) -> Option { + let head = fail_iter!(self.inner.next()?); + + // As long as items are the same key, ignore them + fail_iter!(self.drain_key_min(&head.key.user_key)); + + Some(Ok(head)) + } +} + +impl>> DoubleEndedIterator + for MvccStream +{ + fn next_back(&mut self) -> Option { + loop { + let tail = fail_iter!(self.inner.next_back()?); + + let prev = match self.inner.peek_back() { + Some(Ok(prev)) => prev, + Some(Err(_)) => { + #[expect( + clippy::expect_used, + reason = "we just asserted, the peeked value is an error" + )] + return Some(Err(self + .inner + .next_back() + .expect("should exist") + .expect_err("should be error"))); + } + None => { + return Some(Ok(tail)); + } + }; + + if prev.key.user_key < tail.key.user_key { + return Some(Ok(tail)); + } + } + } +} + +#[cfg(test)] +#[expect(clippy::string_lit_as_bytes)] +mod tests { + use super::*; + use crate::{value::InternalValue, ValueType}; + use test_log::test; + + macro_rules! stream { + ($($key:expr, $sub_key:expr, $value_type:expr),* $(,)?) => {{ + let mut values = Vec::new(); + let mut counters = std::collections::HashMap::new(); + + $( + let key = $key.as_bytes(); + let sub_key = $sub_key.as_bytes(); + let value_type = match $value_type { + "V" => ValueType::Value, + "T" => ValueType::Tombstone, + "W" => ValueType::WeakTombstone, + _ => panic!("Unknown value type"), + }; + + let counter = counters.entry($key).and_modify(|x| { *x -= 1 }).or_insert(999); + values.push(InternalValue::from_components(key, sub_key, *counter, value_type)); + )* + + values + }}; + } + + macro_rules! iter_closed { + ($iter:expr) => { + assert!($iter.next().is_none(), "iterator should be closed (done)"); + assert!( + $iter.next_back().is_none(), + "iterator should be closed (done)" + ); + }; + } + + /// Tests that the iterator emit the same stuff forwards and backwards, just in reverse + macro_rules! test_reverse { + ($v:expr) => { + let iter = Box::new($v.iter().cloned().map(Ok)); + let iter = MvccStream::new(iter); + let mut forwards = iter.flatten().collect::>(); + forwards.reverse(); + + let iter = Box::new($v.iter().cloned().map(Ok)); + let iter = MvccStream::new(iter); + let backwards = iter.rev().flatten().collect::>(); + + assert_eq!(forwards, backwards); + }; + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_error() -> crate::Result<()> { + { + let vec = [ + Ok(InternalValue::from_components( + "a", + "new", + 999, + ValueType::Value, + )), + Err(crate::Error::Io(std::io::Error::other("test error"))), + ]; + + let iter = Box::new(vec.into_iter()); + let mut iter = MvccStream::new(iter); + + // Because next calls drain_key_min, the error is immediately first, even though + // the first item is technically Ok + assert!(matches!(iter.next().unwrap(), Err(crate::Error::Io(_)))); + iter_closed!(iter); + } + + { + let vec = [ + Ok(InternalValue::from_components( + "a", + "new", + 999, + ValueType::Value, + )), + Err(crate::Error::Io(std::io::Error::other("test error"))), + ]; + + let iter = Box::new(vec.into_iter()); + let mut iter = MvccStream::new(iter); + + assert!(matches!( + iter.next_back().unwrap(), + Err(crate::Error::Io(_)) + )); + assert_eq!( + InternalValue::from_components(*b"a", *b"new", 999, ValueType::Value), + iter.next_back().unwrap()?, + ); + iter_closed!(iter); + } + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_queue_reverse_almost_gone() -> crate::Result<()> { + let vec = [ + InternalValue::from_components("a", "a", 0, ValueType::Value), + InternalValue::from_components("b", "", 1, ValueType::Tombstone), + InternalValue::from_components("b", "b", 0, ValueType::Value), + InternalValue::from_components("c", "", 1, ValueType::Tombstone), + InternalValue::from_components("c", "c", 0, ValueType::Value), + InternalValue::from_components("d", "", 1, ValueType::Tombstone), + InternalValue::from_components("d", "d", 0, ValueType::Value), + InternalValue::from_components("e", "", 1, ValueType::Tombstone), + InternalValue::from_components("e", "e", 0, ValueType::Value), + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"a", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"d", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"e", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_queue_almost_gone_2() -> crate::Result<()> { + let vec = [ + InternalValue::from_components("a", "a", 0, ValueType::Value), + InternalValue::from_components("b", "", 1, ValueType::Tombstone), + InternalValue::from_components("c", "", 1, ValueType::Tombstone), + InternalValue::from_components("d", "", 1, ValueType::Tombstone), + InternalValue::from_components("e", "", 1, ValueType::Tombstone), + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"a", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"d", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"e", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_queue() -> crate::Result<()> { + let vec = [ + InternalValue::from_components("a", "a", 0, ValueType::Value), + InternalValue::from_components("b", "b", 0, ValueType::Value), + InternalValue::from_components("c", "c", 0, ValueType::Value), + InternalValue::from_components("d", "d", 0, ValueType::Value), + InternalValue::from_components("e", "", 1, ValueType::Tombstone), + InternalValue::from_components("e", "e", 0, ValueType::Value), + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"a", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"b", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"c", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"d", *b"d", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"e", *b"", 1, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_queue_weak_almost_gone() -> crate::Result<()> { + let vec = [ + InternalValue::from_components("a", "a", 0, ValueType::Value), + InternalValue::from_components("b", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("b", "b", 0, ValueType::Value), + InternalValue::from_components("c", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("c", "c", 0, ValueType::Value), + InternalValue::from_components("d", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("d", "d", 0, ValueType::Value), + InternalValue::from_components("e", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("e", "e", 0, ValueType::Value), + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"a", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"d", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"e", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_queue_weak_almost_gone_2() -> crate::Result<()> { + let vec = [ + InternalValue::from_components("a", "a", 0, ValueType::Value), + InternalValue::from_components("b", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("c", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("d", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("e", "", 1, ValueType::WeakTombstone), + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"a", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"d", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"e", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_queue_weak_reverse() -> crate::Result<()> { + let vec = [ + InternalValue::from_components("a", "a", 0, ValueType::Value), + InternalValue::from_components("b", "b", 0, ValueType::Value), + InternalValue::from_components("c", "c", 0, ValueType::Value), + InternalValue::from_components("d", "d", 0, ValueType::Value), + InternalValue::from_components("e", "", 1, ValueType::WeakTombstone), + InternalValue::from_components("e", "e", 0, ValueType::Value), + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"a", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"b", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"c", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"d", *b"d", 0, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"e", *b"", 1, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_simple() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "new", "V", + "a", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"new", 999, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_simple_multi_keys() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "new", "V", + "a", "old", "V", + "b", "new", "V", + "b", "old", "V", + "c", "newnew", "V", + "c", "new", "V", + "c", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"new", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"new", 999, ValueType::Value), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"newnew", 999, ValueType::Value), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_tombstone() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "T", + "a", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_tombstone_multi_keys() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "T", + "a", "old", "V", + "b", "", "T", + "b", "old", "V", + "c", "", "T", + "c", "", "T", + "c", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_weak_tombstone_simple() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_weak_tombstone_resurrection() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "new", "V", + "a", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_weak_tombstone_priority() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "T", + "a", "", "W", + "a", "new", "V", + "a", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::Tombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn mvcc_stream_weak_tombstone_multi_keys() -> crate::Result<()> { + #[rustfmt::skip] + let vec = stream![ + "a", "", "W", + "a", "old", "V", + "b", "", "W", + "b", "old", "V", + "c", "", "W", + "c", "old", "V", + ]; + + let iter = Box::new(vec.iter().cloned().map(Ok)); + + let mut iter = MvccStream::new(iter); + + assert_eq!( + InternalValue::from_components(*b"a", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"b", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + assert_eq!( + InternalValue::from_components(*b"c", *b"", 999, ValueType::WeakTombstone), + iter.next().unwrap()?, + ); + iter_closed!(iter); + + test_reverse!(vec); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/path.rs b/crates/lsm-tree/src/path.rs new file mode 100644 index 000000000..18740f5df --- /dev/null +++ b/crates/lsm-tree/src/path.rs @@ -0,0 +1,11 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::path::{Path, PathBuf}; + +pub fn absolute_path(path: &Path) -> PathBuf { + // Not sure if this can even fail realistically + #[expect(clippy::expect_used, reason = "not much we can do about it")] + std::path::absolute(path).expect("should be absolute path") +} diff --git a/crates/lsm-tree/src/range.rs b/crates/lsm-tree/src/range.rs new file mode 100644 index 000000000..c0cd5df94 --- /dev/null +++ b/crates/lsm-tree/src/range.rs @@ -0,0 +1,298 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + key::InternalKey, + memtable::Memtable, + merge::Merger, + mvcc_stream::MvccStream, + run_reader::RunReader, + value::{SeqNo, UserKey}, + version::SuperVersion, + BoxedIterator, InternalValue, +}; +use self_cell::self_cell; +use std::{ + ops::{Bound, RangeBounds}, + sync::Arc, +}; + +#[must_use] +pub fn seqno_filter(item_seqno: SeqNo, seqno: SeqNo) -> bool { + item_seqno < seqno +} + +/// Calculates the prefix's upper range. +/// +/// # Panics +/// +/// Panics if the prefix is empty. +pub(crate) fn prefix_upper_range(prefix: &[u8]) -> Bound { + use std::ops::Bound::{Excluded, Unbounded}; + + assert!(!prefix.is_empty(), "prefix may not be empty"); + + let mut end = prefix.to_vec(); + let len = end.len(); + + for (idx, byte) in end.iter_mut().rev().enumerate() { + let idx = len - 1 - idx; + + if *byte < 255 { + *byte += 1; + end.truncate(idx + 1); + return Excluded(end.into()); + } + } + + Unbounded +} + +/// Converts a prefix to range bounds. +#[must_use] +#[expect(clippy::module_name_repetitions)] +pub fn prefix_to_range(prefix: &[u8]) -> (Bound, Bound) { + use std::ops::Bound::{Included, Unbounded}; + + if prefix.is_empty() { + return (Unbounded, Unbounded); + } + + (Included(prefix.into()), prefix_upper_range(prefix)) +} + +/// The iter state references the memtables used while the range is open +/// +/// Because of Rust rules, the state is referenced using `self_cell`, see below. +pub struct IterState { + pub(crate) version: SuperVersion, + pub(crate) ephemeral: Option<(Arc, SeqNo)>, +} + +type BoxedMerge<'a> = Box> + Send + 'a>; + +self_cell!( + pub struct TreeIter { + owner: IterState, + + #[covariant] + dependent: BoxedMerge, + } +); + +impl Iterator for TreeIter { + type Item = crate::Result; + + fn next(&mut self) -> Option { + self.with_dependent_mut(|_, iter| iter.next()) + } +} + +impl DoubleEndedIterator for TreeIter { + fn next_back(&mut self) -> Option { + self.with_dependent_mut(|_, iter| iter.next_back()) + } +} + +impl TreeIter { + pub fn create_range, R: RangeBounds>( + guard: IterState, + range: R, + seqno: SeqNo, + ) -> Self { + Self::new(guard, |lock| { + let lo = match range.start_bound() { + // NOTE: See memtable.rs for range explanation + Bound::Included(key) => Bound::Included(InternalKey::new( + key.as_ref(), + SeqNo::MAX, + crate::ValueType::Tombstone, + )), + Bound::Excluded(key) => Bound::Excluded(InternalKey::new( + key.as_ref(), + 0, + crate::ValueType::Tombstone, + )), + Bound::Unbounded => Bound::Unbounded, + }; + + let hi = match range.end_bound() { + // NOTE: See memtable.rs for range explanation, this is the reverse case + // where we need to go all the way to the last seqno of an item + // + // Example: We search for (Unbounded..Excluded(abdef)) + // + // key -> seqno + // + // a -> 7 <<< This is the lowest key that matches the range + // abc -> 5 + // abc -> 4 + // abc -> 3 <<< This is the highest key that matches the range + // abcdef -> 6 + // abcdef -> 5 + // + Bound::Included(key) => { + Bound::Included(InternalKey::new(key.as_ref(), 0, crate::ValueType::Value)) + } + Bound::Excluded(key) => Bound::Excluded(InternalKey::new( + key.as_ref(), + SeqNo::MAX, + crate::ValueType::Value, + )), + Bound::Unbounded => Bound::Unbounded, + }; + + let range = (lo, hi); + + let mut iters: Vec> = Vec::with_capacity(5); + + for run in lock + .version + .version + .iter_levels() + .flat_map(|lvl| lvl.iter()) + { + match run.len() { + 0 => { + // Do nothing + } + 1 => { + #[expect(clippy::expect_used, reason = "we checked for length")] + let table = run.first().expect("should exist"); + + if table.check_key_range_overlap(&( + range.start_bound().map(|x| &*x.user_key), + range.end_bound().map(|x| &*x.user_key), + )) { + let reader = table + .range(( + range.start_bound().map(|x| &x.user_key).cloned(), + range.end_bound().map(|x| &x.user_key).cloned(), + )) + .filter(move |item| match item { + Ok(item) => seqno_filter(item.key.seqno, seqno), + Err(_) => true, + }); + + iters.push(Box::new(reader)); + } + } + _ => { + if let Some(reader) = RunReader::new( + run.clone(), + ( + range.start_bound().map(|x| &x.user_key).cloned(), + range.end_bound().map(|x| &x.user_key).cloned(), + ), + ) { + iters.push(Box::new(reader.filter(move |item| match item { + Ok(item) => seqno_filter(item.key.seqno, seqno), + Err(_) => true, + }))); + } + } + } + } + + // Sealed memtables + for memtable in lock.version.sealed_memtables.iter() { + let iter = memtable.range(range.clone()); + + iters.push(Box::new( + iter.filter(move |item| seqno_filter(item.key.seqno, seqno)) + .map(Ok), + )); + } + + // Active memtable + { + let iter = lock.version.active_memtable.range(range.clone()); + + iters.push(Box::new( + iter.filter(move |item| seqno_filter(item.key.seqno, seqno)) + .map(Ok), + )); + } + + if let Some((mt, seqno)) = &lock.ephemeral { + let iter = Box::new( + mt.range(range) + .filter(move |item| seqno_filter(item.key.seqno, *seqno)) + .map(Ok), + ); + iters.push(iter); + } + + let merged = Merger::new(iters); + let iter = MvccStream::new(merged); + + Box::new(iter.filter(|x| match x { + Ok(value) => !value.key.is_tombstone(), + Err(_) => true, + })) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Slice; + use std::ops::Bound::{Excluded, Included, Unbounded}; + use test_log::test; + + fn test_prefix(prefix: &[u8], upper_bound: Bound<&[u8]>) { + let range = prefix_to_range(prefix); + assert_eq!( + range, + ( + match prefix { + _ if prefix.is_empty() => Unbounded, + _ => Included(Slice::from(prefix)), + }, + upper_bound.map(Slice::from), + ), + ); + } + + #[test] + fn prefix_to_range_basic() { + test_prefix(b"abc", Excluded(b"abd")); + } + + #[test] + fn prefix_to_range_empty() { + test_prefix(b"", Unbounded); + } + + #[test] + fn prefix_to_range_single_char() { + test_prefix(b"a", Excluded(b"b")); + } + + #[test] + fn prefix_to_range_1() { + test_prefix(&[0, 250], Excluded(&[0, 251])); + } + + #[test] + fn prefix_to_range_2() { + test_prefix(&[0, 250, 50], Excluded(&[0, 250, 51])); + } + + #[test] + fn prefix_to_range_3() { + test_prefix(&[255, 255, 255], Unbounded); + } + + #[test] + fn prefix_to_range_char_max() { + test_prefix(&[0, 255], Excluded(&[1])); + } + + #[test] + fn prefix_to_range_char_max_2() { + test_prefix(&[0, 2, 255], Excluded(&[0, 3])); + } +} diff --git a/crates/lsm-tree/src/run_reader.rs b/crates/lsm-tree/src/run_reader.rs new file mode 100644 index 000000000..96384e66b --- /dev/null +++ b/crates/lsm-tree/src/run_reader.rs @@ -0,0 +1,325 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{version::Run, BoxedIterator, InternalValue, Table, UserKey}; +use std::{ + ops::{Deref, RangeBounds}, + sync::Arc, +}; + +/// Reads through a disjoint run +pub struct RunReader { + run: Arc>, + lo: usize, + hi: usize, + lo_reader: Option>, + hi_reader: Option>, +} + +impl RunReader { + #[must_use] + pub fn new + Clone + Send + 'static>( + run: Arc>, + range: R, + ) -> Option { + assert!(!run.is_empty(), "level reader cannot read empty level"); + + let (lo, hi) = run.range_overlap_indexes(&range)?; + + Some(Self::culled(run, range, (Some(lo), Some(hi)))) + } + + #[must_use] + pub fn culled + Clone + Send + 'static>( + run: Arc>, + range: R, + (lo, hi): (Option, Option), + ) -> Self { + let lo = lo.unwrap_or_default(); + let hi = hi.unwrap_or(run.len() - 1); + + // TODO: lazily init readers? + #[expect( + clippy::expect_used, + reason = "we trust the caller to pass valid indexes" + )] + let lo_table = run.deref().get(lo).expect("should exist"); + let lo_reader = lo_table.range(range.clone()); + + // TODO: lazily init readers? + let hi_reader = if hi > lo { + #[expect( + clippy::expect_used, + reason = "we trust the caller to pass valid indexes" + )] + let hi_table = run.deref().get(hi).expect("should exist"); + Some(hi_table.range(range)) + } else { + None + }; + + Self { + run, + lo, + hi, + lo_reader: Some(Box::new(lo_reader)), + hi_reader: hi_reader.map(|x| Box::new(x) as BoxedIterator), + } + } +} + +impl Iterator for RunReader { + type Item = crate::Result; + + fn next(&mut self) -> Option { + loop { + if let Some(lo_reader) = &mut self.lo_reader { + if let Some(item) = lo_reader.next() { + return Some(item); + } + + // NOTE: Lo reader is empty, get next one + self.lo_reader = None; + self.lo += 1; + + if self.lo < self.hi { + self.lo_reader = Some(Box::new( + #[expect( + clippy::expect_used, + reason = "hi is at most equal to the last slot; so because 0 <= lo < hi, it must be a valid index" + )] + self.run.get(self.lo).expect("should exist").iter(), + )); + } + } else if let Some(hi_reader) = &mut self.hi_reader { + // NOTE: We reached the hi marker, so consume from it instead + // + // If it returns nothing, it is empty, so we are done + return hi_reader.next(); + } else { + return None; + } + } + } +} + +impl DoubleEndedIterator for RunReader { + fn next_back(&mut self) -> Option { + loop { + if let Some(hi_reader) = &mut self.hi_reader { + if let Some(item) = hi_reader.next_back() { + return Some(item); + } + + // NOTE: Hi reader is empty, get prev one + self.hi_reader = None; + self.hi -= 1; + + if self.lo < self.hi { + self.hi_reader = Some(Box::new( + #[expect( + clippy::expect_used, + reason = "because 0 <= lo <= hi, and hi monotonically decreases, hi must be a valid index" + )] + self.run.get(self.hi).expect("should exist").iter(), + )); + } + } else if let Some(lo_reader) = &mut self.lo_reader { + // NOTE: We reached the lo marker, so consume from it instead + // + // If it returns nothing, it is empty, so we are done + return lo_reader.next_back(); + } else { + return None; + } + } + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::{AbstractTree, SequenceNumberCounter, Slice}; + use test_log::test; + + #[test] + fn run_reader_skip() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + let tree = crate::Config::new( + &tempdir, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let ids = [ + ["a", "b", "c"], + ["d", "e", "f"], + ["g", "h", "i"], + ["j", "k", "l"], + ]; + + for batch in ids { + for id in batch { + tree.insert(id, vec![], 0); + } + tree.flush_active_memtable(0)?; + } + + let tables = tree + .current_version() + .iter_tables() + .cloned() + .collect::>(); + + let level = Arc::new(Run::new(tables).unwrap()); + + assert!(RunReader::new(level.clone(), UserKey::from("y")..=UserKey::from("z"),).is_none()); + + assert!(RunReader::new(level, UserKey::from("y")..).is_none()); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn run_reader_basic() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + let tree = crate::Config::new( + &tempdir, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let ids = [ + ["a", "b", "c"], + ["d", "e", "f"], + ["g", "h", "i"], + ["j", "k", "l"], + ]; + + for batch in ids { + for id in batch { + tree.insert(id, vec![], 0); + } + tree.flush_active_memtable(0)?; + } + + let tables = tree + .current_version() + .iter_tables() + .cloned() + .collect::>(); + + let level = Arc::new(Run::new(tables).unwrap()); + + { + let multi_reader = RunReader::culled(level.clone(), .., (Some(1), None)); + let mut iter = multi_reader.flatten(); + + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + { + let multi_reader = RunReader::new(level.clone(), ..).unwrap(); + + let mut iter = multi_reader.flatten(); + + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + { + let multi_reader = RunReader::new(level.clone(), ..).unwrap(); + + let mut iter = multi_reader.rev().flatten(); + + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + { + let multi_reader = RunReader::new(level.clone(), ..).unwrap(); + + let mut iter = multi_reader.flatten(); + + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"l"), iter.next_back().unwrap().key.user_key); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next_back().unwrap().key.user_key); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next_back().unwrap().key.user_key); + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next_back().unwrap().key.user_key); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next_back().unwrap().key.user_key); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next_back().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + { + let multi_reader = RunReader::new(level.clone(), UserKey::from("g")..).unwrap(); + + let mut iter = multi_reader.flatten(); + + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + { + let multi_reader = RunReader::new(level, UserKey::from("g")..).unwrap(); + + let mut iter = multi_reader.flatten().rev(); + + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/run_scanner.rs b/crates/lsm-tree/src/run_scanner.rs new file mode 100644 index 000000000..90d2ed702 --- /dev/null +++ b/crates/lsm-tree/src/run_scanner.rs @@ -0,0 +1,154 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{table::Scanner, version::Run, InternalValue, Table}; +use std::sync::Arc; + +/// Scans through a disjoint run +/// +/// Optimized for compaction, by using a `TableScanner` instead of `TableReader`. +pub struct RunScanner { + tables: Arc>, + lo: usize, + hi: usize, + lo_reader: Option, +} + +impl RunScanner { + pub fn culled( + run: Arc>, + (lo, hi): (Option, Option), + ) -> crate::Result { + let lo = lo.unwrap_or_default(); + let hi = hi.unwrap_or(run.len() - 1); + + #[expect( + clippy::expect_used, + reason = "we trust the caller to pass valid indexes" + )] + let lo_table = run.get(lo).expect("should exist"); + + let lo_reader = lo_table.scan()?; + + Ok(Self { + tables: run, + lo, + hi, + lo_reader: Some(lo_reader), + }) + } +} + +impl Iterator for RunScanner { + type Item = crate::Result; + + fn next(&mut self) -> Option { + loop { + if let Some(lo_reader) = &mut self.lo_reader { + if let Some(item) = lo_reader.next() { + return Some(item); + } + + // NOTE: Lo reader is empty, get next one + self.lo_reader = None; + self.lo += 1; + + if self.lo <= self.hi { + #[expect( + clippy::expect_used, + reason = "hi is at most equal to the last slot; so because 0 <= lo <= hi, it must be a valid index" + )] + let scanner = + fail_iter!(self.tables.get(self.lo).expect("should exist").scan()); + + self.lo_reader = Some(scanner); + } + } else { + return None; + } + } + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::{AbstractTree, SequenceNumberCounter, Slice}; + use test_log::test; + + #[test] + fn run_scanner_basic() -> crate::Result<()> { + let tempdir = tempfile::tempdir()?; + let tree = crate::Config::new( + &tempdir, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let ids = [ + ["a", "b", "c"], + ["d", "e", "f"], + ["g", "h", "i"], + ["j", "k", "l"], + ]; + + for batch in ids { + for id in batch { + tree.insert(id, vec![], 0); + } + tree.flush_active_memtable(0)?; + } + + let tables = tree + .current_version() + .iter_tables() + .cloned() + .collect::>(); + + let level = Arc::new(Run::new(tables).unwrap()); + + #[expect(clippy::unwrap_used)] + { + let multi_reader = RunScanner::culled(level.clone(), (None, None))?; + + let mut iter = multi_reader.flatten(); + + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + #[expect(clippy::unwrap_used)] + { + let multi_reader = RunScanner::culled(level, (Some(1), None))?; + + let mut iter = multi_reader.flatten(); + + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"j"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"k"), iter.next().unwrap().key.user_key); + assert_eq!(Slice::from(*b"l"), iter.next().unwrap().key.user_key); + assert!(iter.next().is_none()); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/seqno.rs b/crates/lsm-tree/src/seqno.rs new file mode 100644 index 000000000..1fd8fb790 --- /dev/null +++ b/crates/lsm-tree/src/seqno.rs @@ -0,0 +1,106 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::SeqNo; +use std::sync::{ + atomic::{ + AtomicU64, + Ordering::{AcqRel, Acquire, Release}, + }, + Arc, +}; + +/// Thread-safe sequence number generator +/// +/// # Examples +/// +/// ``` +/// # use lsm_tree::{AbstractTree, Config, SequenceNumberCounter}; +/// # +/// # let path = tempfile::tempdir()?; +/// +/// let seqno = SequenceNumberCounter::default(); +/// let visible_seqno = SequenceNumberCounter::default(); +/// +/// let tree = Config::new(path, seqno.clone(), visible_seqno.clone()).open()?; +/// +/// // Do some inserts... +/// for k in [b"a", b"b", b"c"] { +/// let batch_seqno = seqno.next(); +/// tree.insert(k, "abc", batch_seqno); +/// visible_seqno.fetch_max(batch_seqno + 1); +/// } +/// +/// // Create a batch +/// let batch_seqno = seqno.next(); +/// tree.remove("a".as_bytes(), batch_seqno); +/// tree.remove("b".as_bytes(), batch_seqno); +/// tree.remove("c".as_bytes(), batch_seqno); +/// visible_seqno.fetch_max(batch_seqno + 1); +/// # +/// # assert!(tree.is_empty(visible_seqno.get(), None)?); +/// # Ok::<(), lsm_tree::Error>(()) +/// ``` +#[derive(Clone, Default, Debug)] +pub struct SequenceNumberCounter(Arc); + +impl SequenceNumberCounter { + /// Creates a new counter, setting it to some previous value + #[must_use] + pub fn new(prev: SeqNo) -> Self { + Self(Arc::new(AtomicU64::new(prev))) + } + + /// Gets the would-be-next sequence number, without incrementing the counter. + /// + /// This should only be used when creating a snapshot. + #[must_use] + pub fn get(&self) -> SeqNo { + self.0.load(Acquire) + } + + /// Gets the next sequence number. + #[must_use] + #[allow(clippy::missing_panics_doc, reason = "we should never run out of u64s")] + pub fn next(&self) -> SeqNo { + let seqno = self.0.fetch_add(1, AcqRel); + + // The MSB is reserved for transactions. + // + // This gives us 63-bit sequence numbers technically. + assert!(seqno < 0x8000_0000_0000_0000, "Ran out of sequence numbers"); + + seqno + } + + /// Sets the sequence number. + pub fn set(&self, seqno: SeqNo) { + self.0.store(seqno, Release); + } + + /// Maximizes the sequence number. + pub fn fetch_max(&self, seqno: SeqNo) { + self.0.fetch_max(seqno, AcqRel); + } +} + +#[cfg(test)] +mod tests { + use test_log::test; + + #[test] + fn not_max_seqno() { + let counter = super::SequenceNumberCounter::default(); + counter.set(0x7FFF_FFFF_FFFF_FFFF); + let _ = counter.next(); + } + + #[test] + #[should_panic = "Ran out of sequence numbers"] + fn max_seqno() { + let counter = super::SequenceNumberCounter::default(); + counter.set(0x8000_0000_0000_0000); + let _ = counter.next(); + } +} diff --git a/crates/lsm-tree/src/slice/mod.rs b/crates/lsm-tree/src/slice/mod.rs new file mode 100644 index 000000000..63f9c3745 --- /dev/null +++ b/crates/lsm-tree/src/slice/mod.rs @@ -0,0 +1,260 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +// Using tokio bytes +#[cfg(feature = "bytes_1")] +mod slice_bytes; + +// Using byteview +#[cfg(not(feature = "bytes_1"))] +mod slice_default; + +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +#[cfg(not(feature = "bytes_1"))] +pub use slice_default::{Builder, Slice}; + +#[cfg(feature = "bytes_1")] +pub use slice_bytes::{Builder, Slice}; + +impl AsRef<[u8]> for Slice { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl From<&[u8]> for Slice { + fn from(value: &[u8]) -> Self { + #[cfg(not(feature = "bytes_1"))] + { + Self(byteview::ByteView::new(value)) + } + + #[cfg(feature = "bytes_1")] + { + Self(bytes::Bytes::from(value.to_vec())) + } + } +} + +impl From> for Slice { + fn from(value: Arc<[u8]>) -> Self { + Self::from(&*value) + } +} + +impl From<&Vec> for Slice { + fn from(value: &Vec) -> Self { + Self::from(value.as_slice()) + } +} + +impl From<&str> for Slice { + fn from(value: &str) -> Self { + Self::from(value.as_bytes()) + } +} + +impl From<&String> for Slice { + fn from(value: &String) -> Self { + Self::from(value.as_str()) + } +} + +impl From<&Path> for Slice { + fn from(value: &Path) -> Self { + Self::from(value.as_os_str().as_encoded_bytes()) + } +} + +impl From for Slice { + fn from(value: PathBuf) -> Self { + Self::from(value.as_os_str().as_encoded_bytes()) + } +} + +impl From> for Slice { + fn from(value: Arc) -> Self { + Self::from(&*value) + } +} + +impl From<[u8; N]> for Slice { + fn from(value: [u8; N]) -> Self { + Self::from(value.as_slice()) + } +} + +impl From<&[u8; N]> for Slice { + fn from(value: &[u8; N]) -> Self { + Self::from(value.as_slice()) + } +} + +impl FromIterator for Slice { + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Vec::from_iter(iter).into() + } +} + +impl std::ops::Deref for Slice { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + self.as_ref() + } +} + +impl std::borrow::Borrow<[u8]> for Slice { + fn borrow(&self) -> &[u8] { + self + } +} + +impl PartialEq for Slice +where + T: AsRef<[u8]>, +{ + fn eq(&self, other: &T) -> bool { + self.as_ref() == other.as_ref() + } +} + +impl PartialEq for &[u8] { + fn eq(&self, other: &Slice) -> bool { + *self == other.as_ref() + } +} + +impl PartialOrd for Slice +where + T: AsRef<[u8]>, +{ + fn partial_cmp(&self, other: &T) -> Option { + self.as_ref().partial_cmp(other.as_ref()) + } +} + +impl PartialOrd for &[u8] { + fn partial_cmp(&self, other: &Slice) -> Option { + (*self).partial_cmp(other.as_ref()) + } +} + +#[cfg(test)] +#[expect(clippy::expect_used)] +mod tests { + use super::Slice; + use std::{fmt::Debug, sync::Arc}; + use test_log::test; + + fn assert_slice_handles(v: T) + where + T: Clone + Debug, + Slice: From + PartialEq + PartialOrd, + { + // verify slice arc roundtrips + let slice: Slice = v.clone().into(); + assert_eq!(slice, v, "slice_arc: {slice:?}, v: {v:?}"); + assert!(slice >= v, "slice_arc: {slice:?}, v: {v:?}"); + } + + #[test] + fn slice_empty() { + assert_eq!(Slice::empty(), []); + } + + #[test] + fn slice_fuse_empty() { + let bytes = Slice::fused(&[], &[]); + assert_eq!(&*bytes, &[] as &[u8]); + } + + #[test] + fn slice_fuse_one() { + let bytes = Slice::fused(b"abc", &[]); + assert_eq!(&*bytes, b"abc"); + } + + #[test] + fn slice_fuse_two() { + let bytes = Slice::fused(b"abc", b"def"); + assert_eq!(&*bytes, b"abcdef"); + } + + #[test] + #[expect(unsafe_code)] + fn slice_with_size() { + assert_eq!( + &*unsafe { + let mut b = Slice::builder_unzeroed(5); + b.fill(0); + b.freeze() + }, + [0; 5], + ); + assert_eq!( + &*unsafe { + let mut b = Slice::builder_unzeroed(50); + b.fill(0); + b.freeze() + }, + [0; 50], + ); + assert_eq!( + &*unsafe { + let mut b = Slice::builder_unzeroed(50); + b.fill(77); + b.freeze() + }, + [77; 50], + ); + } + + /// This test verifies that we can create a `Slice` from various types and compare a `Slice` with them. + #[test] + fn test_slice_instantiation() { + // - &[u8] + assert_slice_handles::<&[u8]>(&[1, 2, 3, 4]); + // - Arc + assert_slice_handles::>(Arc::new([1, 2, 3, 4])); + // - Vec + assert_slice_handles::>(vec![1, 2, 3, 4]); + // - &str + assert_slice_handles::<&str>("hello"); + // - String + assert_slice_handles::("hello".to_string()); + // - [u8; N] + assert_slice_handles::<[u8; 4]>([1, 2, 3, 4]); + + // Special case for these types + // - Iterator + let slice = Slice::from_iter(vec![1, 2, 3, 4]); + assert_eq!(slice, vec![1, 2, 3, 4]); + + // - Arc + let arc_str: Arc = Arc::from("hello"); + let slice = Slice::from(arc_str.clone()); + assert_eq!(slice.as_ref(), arc_str.as_bytes()); + + // - io::Read + let mut reader = std::io::Cursor::new(vec![1, 2, 3, 4]); + let slice = Slice::from_reader(&mut reader, 4).expect("read"); + assert_eq!(slice, vec![1, 2, 3, 4]); + } + + /// Verifies that `Slice::as_slice` returns the data as-is. + #[test] + fn test_slice_as_slice() { + let original = [1_u8, 2, 3, 4]; + let slice = Slice::from_iter(original.iter().copied()); + assert_eq!(slice.as_slice(), original); + } +} diff --git a/crates/lsm-tree/src/slice/slice_bytes/mod.rs b/crates/lsm-tree/src/slice/slice_bytes/mod.rs new file mode 100644 index 000000000..936fe423d --- /dev/null +++ b/crates/lsm-tree/src/slice/slice_bytes/mod.rs @@ -0,0 +1,115 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use bytes::Bytes; + +pub use bytes::BytesMut as Builder; + +/// An immutable byte slice that can be cloned without additional heap allocation +/// +/// There is no guarantee of any sort of alignment for zero-copy (de)serialization. +#[derive(Debug, Default, Clone, Eq, Hash, Ord)] +pub struct Slice(pub(super) Bytes); + +impl Slice { + /// Construct a [`Slice`] from a byte slice. + #[must_use] + pub fn new(bytes: &[u8]) -> Self { + Self(Bytes::copy_from_slice(bytes)) + } + + /// Returns the bytes slice containing the entire data. + #[must_use] + pub fn as_slice(&self) -> &[u8] { + self + } + + #[doc(hidden)] + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + #[doc(hidden)] + pub unsafe fn builder_unzeroed(len: usize) -> Builder { + // Use `with_capacity` & `set_len`` to avoid zeroing the buffer + let mut builder = Builder::with_capacity(len); + + // SAFETY: we just allocated `len` bytes, and `read_exact` will fail if + // it doesn't fill the buffer, subsequently dropping the uninitialized + // BytesMut object + #[expect(unsafe_code, reason = "see safety")] + unsafe { + builder.set_len(len); + } + + builder + } + + #[doc(hidden)] + #[must_use] + pub fn slice(&self, range: impl std::ops::RangeBounds) -> Self { + Self(self.0.slice(range)) + } + + #[doc(hidden)] + #[must_use] + pub fn fused(left: &[u8], right: &[u8]) -> Self { + use std::io::Write; + + let len = left.len() + right.len(); + let mut builder = unsafe { Self::builder_unzeroed(len) }; + { + let mut writer = &mut builder[..]; + + writer.write_all(left).expect("should write"); + writer.write_all(right).expect("should write"); + } + + Self(builder.freeze()) + } + + /// Constructs a [`Slice`] from an I/O reader by pulling in `len` bytes. + /// + /// The reader may not read the existing buffer. + #[doc(hidden)] + pub fn from_reader(reader: &mut R, len: usize) -> std::io::Result { + let mut builder = unsafe { Self::builder_unzeroed(len) }; + + // SAFETY: Normally, read_exact over an uninitialized buffer is UB, + // however we know that in lsm-tree etc. only I/O readers or cursors over Vecs are used + // so it's safe + // + // The safe API is unstable: https://github.com/rust-lang/rust/issues/78485 + reader.read_exact(&mut builder)?; + + Ok(Self(builder.freeze())) + } +} + +impl From for Slice { + fn from(value: Bytes) -> Self { + Self(value) + } +} + +impl From for Bytes { + fn from(value: Slice) -> Self { + value.0 + } +} + +// Bytes::from> is zero-copy optimized +impl From> for Slice { + fn from(value: Vec) -> Self { + Self(Bytes::from(value)) + } +} + +// Bytes::from is zero-copy optimized +impl From for Slice { + fn from(value: String) -> Self { + Self(Bytes::from(value)) + } +} diff --git a/crates/lsm-tree/src/slice/slice_default/mod.rs b/crates/lsm-tree/src/slice/slice_default/mod.rs new file mode 100644 index 000000000..2c9c291a9 --- /dev/null +++ b/crates/lsm-tree/src/slice/slice_default/mod.rs @@ -0,0 +1,78 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use byteview::ByteView; + +pub use byteview::Builder; + +/// An immutable byte slice that can be cloned without additional heap allocation +/// +/// There is no guarantee of any sort of alignment for zero-copy (de)serialization. +#[derive(Debug, Default, Clone, Eq, Hash, Ord)] +pub struct Slice(pub(super) ByteView); + +impl Slice { + /// Construct a [`Slice`] from a byte slice. + #[must_use] + pub fn new(bytes: &[u8]) -> Self { + Self(bytes.into()) + } + + /// Returns the bytes slice containing the entire data. + #[must_use] + pub fn as_slice(&self) -> &[u8] { + self + } + + #[doc(hidden)] + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + #[doc(hidden)] + #[must_use] + pub unsafe fn builder_unzeroed(len: usize) -> Builder { + unsafe { ByteView::builder_unzeroed(len) } + } + + pub(crate) fn slice(&self, range: impl std::ops::RangeBounds) -> Self { + Self(self.0.slice(range)) + } + + pub(crate) fn fused(left: &[u8], right: &[u8]) -> Self { + Self(ByteView::fused(left, right)) + } + + #[doc(hidden)] + pub fn from_reader(reader: &mut R, len: usize) -> std::io::Result { + ByteView::from_reader(reader, len).map(Self) + } +} + +// Arc::from> is specialized +impl From> for Slice { + fn from(value: Vec) -> Self { + Self(ByteView::from(value)) + } +} + +// Arc::from> is specialized +impl From for Slice { + fn from(value: String) -> Self { + Self(ByteView::from(value.into_bytes())) + } +} + +impl From for Slice { + fn from(value: ByteView) -> Self { + Self(value) + } +} + +impl From for ByteView { + fn from(value: Slice) -> Self { + value.0 + } +} diff --git a/crates/lsm-tree/src/slice_windows.rs b/crates/lsm-tree/src/slice_windows.rs new file mode 100644 index 000000000..a1f2e8bce --- /dev/null +++ b/crates/lsm-tree/src/slice_windows.rs @@ -0,0 +1,94 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub trait GrowingWindowsExt { + fn growing_windows<'a>(&'a self) -> impl Iterator + where + T: 'a; +} + +impl GrowingWindowsExt for [T] { + fn growing_windows<'a>(&'a self) -> impl Iterator + where + T: 'a, + { + (1..=self.len()).flat_map(|size| self.windows(size)) + } +} + +pub trait ShrinkingWindowsExt { + fn shrinking_windows<'a>(&'a self) -> impl Iterator + where + T: 'a; +} + +impl ShrinkingWindowsExt for [T] { + fn shrinking_windows<'a>(&'a self) -> impl Iterator + where + T: 'a, + { + (1..=self.len()).rev().flat_map(|size| self.windows(size)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[expect(clippy::unwrap_used)] + fn test_growing_windows() { + let a = [1, 2, 3, 4, 5]; + + let mut windows = a.growing_windows(); + + assert_eq!(&[1], windows.next().unwrap()); + assert_eq!(&[2], windows.next().unwrap()); + assert_eq!(&[3], windows.next().unwrap()); + assert_eq!(&[4], windows.next().unwrap()); + assert_eq!(&[5], windows.next().unwrap()); + + assert_eq!(&[1, 2], windows.next().unwrap()); + assert_eq!(&[2, 3], windows.next().unwrap()); + assert_eq!(&[3, 4], windows.next().unwrap()); + assert_eq!(&[4, 5], windows.next().unwrap()); + + assert_eq!(&[1, 2, 3], windows.next().unwrap()); + assert_eq!(&[2, 3, 4], windows.next().unwrap()); + assert_eq!(&[3, 4, 5], windows.next().unwrap()); + + assert_eq!(&[1, 2, 3, 4], windows.next().unwrap()); + assert_eq!(&[2, 3, 4, 5], windows.next().unwrap()); + + assert_eq!(&[1, 2, 3, 4, 5], windows.next().unwrap()); + } + + #[test] + #[expect(clippy::unwrap_used)] + fn test_shrinking_windows() { + let a = [1, 2, 3, 4, 5]; + + let mut windows = a.shrinking_windows(); + + assert_eq!(&[1, 2, 3, 4, 5], windows.next().unwrap()); + + assert_eq!(&[1, 2, 3, 4], windows.next().unwrap()); + assert_eq!(&[2, 3, 4, 5], windows.next().unwrap()); + + assert_eq!(&[1, 2, 3], windows.next().unwrap()); + assert_eq!(&[2, 3, 4], windows.next().unwrap()); + assert_eq!(&[3, 4, 5], windows.next().unwrap()); + + assert_eq!(&[1, 2], windows.next().unwrap()); + assert_eq!(&[2, 3], windows.next().unwrap()); + assert_eq!(&[3, 4], windows.next().unwrap()); + assert_eq!(&[4, 5], windows.next().unwrap()); + + assert_eq!(&[1], windows.next().unwrap()); + assert_eq!(&[2], windows.next().unwrap()); + assert_eq!(&[3], windows.next().unwrap()); + assert_eq!(&[4], windows.next().unwrap()); + assert_eq!(&[5], windows.next().unwrap()); + } +} diff --git a/crates/lsm-tree/src/stop_signal.rs b/crates/lsm-tree/src/stop_signal.rs new file mode 100644 index 000000000..8d83cfaab --- /dev/null +++ b/crates/lsm-tree/src/stop_signal.rs @@ -0,0 +1,19 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use std::sync::{atomic::AtomicBool, Arc}; + +#[derive(Clone, Debug, Default)] +pub struct StopSignal(Arc); + +impl StopSignal { + pub fn send(&self) { + self.0.store(true, std::sync::atomic::Ordering::Release); + } + + #[must_use] + pub fn is_stopped(&self) -> bool { + self.0.load(std::sync::atomic::Ordering::Acquire) + } +} diff --git a/crates/lsm-tree/src/table/block/binary_index/builder.rs b/crates/lsm-tree/src/table/block/binary_index/builder.rs new file mode 100644 index 000000000..dcd0d6475 --- /dev/null +++ b/crates/lsm-tree/src/table/block/binary_index/builder.rs @@ -0,0 +1,55 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use byteorder::{LittleEndian, WriteBytesExt}; + +#[derive(Debug)] +pub struct Builder(Vec); + +impl Builder { + pub fn new(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + pub fn insert(&mut self, pos: u32) { + self.0.push(pos); + } + + pub fn write(&self, writer: &mut W) -> crate::Result<(u8, usize)> { + // NOTE: We check if the pointers may fit in 16-bits + // If so, we halve the index size by storing u16 instead of u32 + let step_size = { + #[expect( + clippy::expect_used, + reason = "blocks are never empty, so the binary seek index cannot be empty either" + )] + if u16::try_from(*self.0.last().expect("should not be empty")).is_ok() { + 2 + } else { + 4 + } + }; + + let len = self.0.len(); + + if step_size == 2 { + // Write u16 index + for &offset in &self.0 { + #[expect( + clippy::cast_possible_truncation, + reason = "above we check that the last (highest) pointer still fits into u16" + )] + let offset = offset as u16; + writer.write_u16::(offset)?; + } + } else { + // Write u32 index + for &offset in &self.0 { + writer.write_u32::(offset)?; + } + } + + Ok((step_size, len)) + } +} diff --git a/crates/lsm-tree/src/table/block/binary_index/mod.rs b/crates/lsm-tree/src/table/block/binary_index/mod.rs new file mode 100644 index 000000000..bdd12251d --- /dev/null +++ b/crates/lsm-tree/src/table/block/binary_index/mod.rs @@ -0,0 +1,9 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod builder; +mod reader; + +pub use builder::Builder; +pub use reader::Reader; diff --git a/crates/lsm-tree/src/table/block/binary_index/reader.rs b/crates/lsm-tree/src/table/block/binary_index/reader.rs new file mode 100644 index 000000000..09fa6c060 --- /dev/null +++ b/crates/lsm-tree/src/table/block/binary_index/reader.rs @@ -0,0 +1,49 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use byteorder::{LittleEndian, ReadBytesExt}; + +pub struct Reader<'a> { + bytes: &'a [u8], + step_size: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(bytes: &'a [u8], offset: u32, len: u32, step_size: u8) -> Self { + let offset = offset as usize; + let len = len as usize; + let step_size = step_size as usize; + let size = len * step_size; + let end = offset + size; + + Self { + #[expect( + clippy::indexing_slicing, + reason = "we consider the caller to be trustworthy" + )] + bytes: &bytes[offset..end], + step_size, + } + } + + pub fn len(&self) -> usize { + self.bytes.len() / self.step_size + } + + pub(crate) fn get(&self, idx: usize) -> usize { + let offset = idx * self.step_size; + + #[expect( + clippy::indexing_slicing, + reason = "we consider the caller to be trustworthy" + )] + let mut bytes = &self.bytes[offset..]; + + if self.step_size == 2 { + unwrap!(bytes.read_u16::()).into() + } else { + unwrap!(bytes.read_u32::()) as usize + } + } +} diff --git a/crates/lsm-tree/src/table/block/decoder.rs b/crates/lsm-tree/src/table/block/decoder.rs new file mode 100644 index 000000000..04d275a61 --- /dev/null +++ b/crates/lsm-tree/src/table/block/decoder.rs @@ -0,0 +1,558 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::binary_index::Reader as BinaryIndexReader; +use crate::{ + SeqNo, Slice, + table::{Block, block::Trailer}, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::{io::Cursor, marker::PhantomData}; + +/// Represents an object that was parsed from a byte array +/// +/// Parsed items only hold references to their keys and values, use `materialize` to create an owned value. +pub trait ParsedItem { + /// Compares this item's key with a needle. + /// + /// We can not access the key directly because it may be comprised of prefix + suffix. + fn compare_key(&self, needle: &[u8], bytes: &[u8]) -> std::cmp::Ordering; + + /// Returns the byte offset of the key's start position. + fn key_offset(&self) -> usize; + + /// Converts the parsed representation to an owned value. + fn materialize(&self, bytes: &Slice) -> M; +} + +/// Describes an object that can be parsed from a block, either a full item (restart head), or a truncated item +pub trait Decodable { + /// Parses the key of the next restart head from a reader. + /// + /// This is used for the binary search index. + fn parse_restart_key<'a>( + reader: &mut Cursor<&[u8]>, + offset: usize, + data: &'a [u8], + fixed_key_len: Option, + fixed_value_len: Option, + ) -> Option<(&'a [u8], SeqNo)>; + + /// Parses a restart head from a reader. + /// + /// `offset` is the position of the item to read in the block's byte slice. + fn parse_full( + reader: &mut Cursor<&[u8]>, + offset: usize, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> Option; + + /// Parses a (possibly) prefix truncated item from a reader. + fn parse_truncated( + reader: &mut Cursor<&[u8]>, + offset: usize, + base_key_offset: usize, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> Option; +} + +#[derive(Debug)] +struct LoScanner { + offset: usize, + remaining_in_interval: usize, + base_key_offset: Option, +} + +#[derive(Debug)] +struct HiScanner { + offset: usize, + ptr_idx: usize, + stack: Vec, // TODO: SmallVec? + base_key_offset: Option, +} + +/// Generic block decoder for RocksDB-style blocks +/// +/// Supports prefix truncation and binary search index (through restart intervals). +pub struct Decoder<'a, Item: Decodable, Parsed: ParsedItem> { + block: &'a Block, + phantom: PhantomData<(Item, Parsed)>, + + lo_scanner: LoScanner, + hi_scanner: HiScanner, + + // Cached metadata + restart_interval: u8, + binary_index_step_size: u8, + binary_index_offset: u32, + binary_index_len: u32, + fixed_key_len: Option, + fixed_value_len: Option, +} + +impl<'a, Item: Decodable, Parsed: ParsedItem> Decoder<'a, Item, Parsed> { + #[must_use] + pub fn new(block: &'a Block) -> Self { + let trailer = Trailer::new(block); + let mut reader = trailer.as_slice(); + + let restart_interval = unwrap!(reader.read_u8()); + + let binary_index_step_size = unwrap!(reader.read_u8()); + + debug_assert!( + binary_index_step_size == 2 || binary_index_step_size == 4, + "invalid binary index step size", + ); + + let binary_index_len = unwrap!(reader.read_u32::()); + let binary_index_offset = unwrap!(reader.read_u32::()); + let _hash_index_len = unwrap!(reader.read_u32::()); + let _hash_index_offset = unwrap!(reader.read_u32::()); + let _prefix_truncation = unwrap!(reader.read_u8()); + + let has_fixed_key_len = unwrap!(reader.read_u8()) != 0; + let fixed_key_len = unwrap!(reader.read_u16::()); + let fixed_key_len = has_fixed_key_len.then_some(fixed_key_len); + + let has_fixed_value_len = unwrap!(reader.read_u8()) != 0; + let fixed_value_len = unwrap!(reader.read_u32::()); + let fixed_value_len = has_fixed_value_len.then_some(fixed_value_len); + + Self { + block, + phantom: PhantomData, + + lo_scanner: LoScanner { + offset: 0, + remaining_in_interval: 0, + base_key_offset: None, + }, + + hi_scanner: HiScanner { + offset: 0, + ptr_idx: binary_index_len as usize, + stack: Vec::new(), + base_key_offset: None, + }, + + restart_interval, + + binary_index_step_size, + binary_index_offset, + binary_index_len, + fixed_key_len, + fixed_value_len, + } + } + + fn get_binary_index_reader(&self) -> BinaryIndexReader<'_> { + BinaryIndexReader::new( + &self.block.data, + self.binary_index_offset, + self.binary_index_len, + self.binary_index_step_size, + ) + } + + fn get_key_at(&self, pos: usize) -> (&[u8], SeqNo) { + let bytes = &self.block.data; + + #[expect( + unsafe_code, + reason = "pos is always retrieved from the binary index which we consider to be trustworthy" + )] + let mut cursor = Cursor::new(unsafe { bytes.get_unchecked(pos..) }); + + #[expect( + clippy::expect_used, + reason = "restart key should be parsed, see reason above" + )] + Item::parse_restart_key( + &mut cursor, + pos, + bytes, + self.fixed_key_len, + self.fixed_value_len, + ) + .expect("should parse restart key") + } + + fn partition_point(&self, pred: F) -> Option<(/* offset */ usize, /* idx */ usize)> + where + F: Fn(&[u8], SeqNo) -> bool, + { + // The first pass over the binary index emulates `Iterator::partition_point` over the + // restart heads that are in natural key order. We keep track of both the byte offset and + // the restart index because callers need the offset to seed the linear scanner, while the + // index is sometimes reused (for example by `seek_upper`). + // + // In contrast to the usual `partition_point`, we intentionally return the *last* restart + // entry when the predicate continues to hold for every head key. Forward scans rely on + // this behaviour to land on the final restart interval and resume the linear scan there + // instead of erroneously reporting "not found". + let binary_index = self.get_binary_index_reader(); + + debug_assert!( + binary_index.len() >= 1, + "binary index should never be empty", + ); + + let mut left: usize = 0; + let mut right = binary_index.len(); + + if right == 0 { + return None; + } + + while left < right { + let mid = usize::midpoint(left, right); + + let offset = binary_index.get(mid); + + let (head_key, head_seqno) = self.get_key_at(offset); + + if pred(head_key, head_seqno) { + left = mid + 1; + } else { + right = mid; + } + } + + if left == 0 { + return Some((0, 0)); + } + + if left == binary_index.len() { + let idx = binary_index.len() - 1; + let offset = binary_index.get(idx); + return Some((offset, idx)); + } + + let offset = binary_index.get(left - 1); + + Some((offset, left - 1)) + } + + // TODO: + fn partition_point_2(&self, pred: F) -> Option<(/* offset */ usize, /* idx */ usize)> + where + F: Fn(&[u8], SeqNo) -> bool, + { + // `partition_point_2` mirrors `partition_point` but keeps the *next* restart entry instead + // of the previous one. This variant is used exclusively by reverse scans (`seek_upper`) + // that want the first restart whose head key exceeds the predicate. Returning the raw + // offset preserves the ability to reuse linear scanning infrastructure without duplicating + // decoder logic. + let binary_index = self.get_binary_index_reader(); + + debug_assert!( + binary_index.len() >= 1, + "binary index should never be empty", + ); + + let mut left: usize = 0; + let mut right = binary_index.len(); + + if right == 0 { + return None; + } + + while left < right { + let mid = usize::midpoint(left, right); + + let offset = binary_index.get(mid); + + let (head_key, head_seqno) = self.get_key_at(offset); + + if pred(head_key, head_seqno) { + left = mid + 1; + } else { + right = mid; + } + } + + if left == binary_index.len() { + let idx = binary_index.len() - 1; + let offset = binary_index.get(idx); + return Some((offset, idx)); + } + + let offset = binary_index.get(left); + + Some((offset, left)) + } + + pub fn set_lo_offset(&mut self, offset: usize) { + self.lo_scanner.offset = offset; + } + + /// Seeks using the given predicate. + /// + /// Returns `false` if the key does not possible exist. + pub fn seek(&mut self, pred: impl Fn(&[u8], SeqNo) -> bool, second_partition: bool) -> bool { + // TODO: make this nicer, maybe predicate that can affect the resulting index...? + let result = if second_partition { + self.partition_point_2(&pred) + } else { + self.partition_point(&pred) + }; + + // Binary index lookup + let Some((offset, _)) = result else { + return false; + }; + + if second_partition && self.restart_interval == 1 && { + let (key, seqno) = self.get_key_at(offset); + pred(key, seqno) + } { + // `second_partition == true` means we ran the "look one restart ahead" search used by + // index blocks. When the predicate is still true at the chosen restart head it means + // the caller asked us to seek strictly beyond the last entry. In that case we skip any + // costly parsing and flip both scanners into an "exhausted" state so the outer iterator + // immediately reports EOF. + let end = self.block.data.len(); + + self.lo_scanner.offset = end; + self.lo_scanner.remaining_in_interval = 0; + self.lo_scanner.base_key_offset = None; + + self.hi_scanner.offset = end; + self.hi_scanner.ptr_idx = usize::MAX; + self.hi_scanner.stack.clear(); + self.hi_scanner.base_key_offset = Some(0); + + return false; + } + + self.lo_scanner.offset = offset; + + true + } + + /// Seeks the upper bound using the given predicate. + /// + /// Returns `false` if the key does not possible exist. + pub fn seek_upper( + &mut self, + pred: impl Fn(&[u8], SeqNo) -> bool, + second_partition: bool, + ) -> bool { + let result = if second_partition { + self.partition_point_2(&pred) + } else { + self.partition_point(&pred) + }; + + // Binary index lookup + let Some((offset, idx)) = result else { + return false; + }; + + self.hi_scanner.offset = offset; + self.hi_scanner.ptr_idx = idx; + self.hi_scanner.stack.clear(); + self.hi_scanner.base_key_offset = None; + + self.fill_stack(); + + true + } + + fn parse_current_item( + reader: &mut Cursor<&[u8]>, + offset: usize, + base_key_offset: Option, + is_restart: bool, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> Option { + if is_restart { + Item::parse_full(reader, offset, fixed_key_len, fixed_value_len) + } else { + #[expect(clippy::expect_used, reason = "we trust the is_restart flag")] + Item::parse_truncated( + reader, + offset, + base_key_offset.expect("should parse truncated item"), + fixed_key_len, + fixed_value_len, + ) + } + } + + fn fill_stack(&mut self) { + let binary_index = self.get_binary_index_reader(); + + { + self.hi_scanner.offset = binary_index.get(self.hi_scanner.ptr_idx); + + let offset = self.hi_scanner.offset; + + #[expect( + unsafe_code, + reason = "cursor is advanced by read_ operations which check for EOF, and the cursor starts at 0 - the slice is never empty" + )] + let mut reader = Cursor::new(unsafe { self.block.data.get_unchecked(offset..) }); + + #[expect( + clippy::cast_possible_truncation, + reason = "blocks do not even come close to 4 GiB in size" + )] + if Item::parse_full( + &mut reader, + offset, + self.fixed_key_len, + self.fixed_value_len, + ) + .inspect(|item| { + self.hi_scanner.offset += reader.position() as usize; + self.hi_scanner.base_key_offset = Some(item.key_offset()); + }) + .is_some() + { + self.hi_scanner.stack.push(offset); + } + } + + for _ in 1..self.restart_interval { + let offset = self.hi_scanner.offset; + + #[expect( + unsafe_code, + reason = "cursor is advanced by read_ operations which check for EOF, and the cursor starts at 0 - the slice is never empty" + )] + let mut reader = Cursor::new(unsafe { self.block.data.get_unchecked(offset..) }); + + #[expect(clippy::expect_used, reason = "base key offset is expected to exist")] + #[expect( + clippy::cast_possible_truncation, + reason = "blocks do not even come close to 4 GiB in size" + )] + if Item::parse_truncated( + &mut reader, + offset, + self.hi_scanner.base_key_offset.expect("should exist"), + self.fixed_key_len, + self.fixed_value_len, + ) + .inspect(|_| { + self.hi_scanner.offset += reader.position() as usize; + }) + .is_some() + { + self.hi_scanner.stack.push(offset); + } else { + break; + } + } + } + + fn consume_stack_top(&mut self) -> Option { + let offset = self.hi_scanner.stack.pop()?; + + if self.lo_scanner.offset > 0 && offset < self.lo_scanner.offset { + return None; + } + + self.hi_scanner.offset = offset; + + let is_restart = self.hi_scanner.stack.is_empty(); + + #[expect( + unsafe_code, + reason = "cursor is advanced by read_ operations which check for EOF, and the cursor starts at 0 - the slice is never empty" + )] + let mut reader = Cursor::new(unsafe { self.block.data.get_unchecked(offset..) }); + + Self::parse_current_item( + &mut reader, + offset, + self.hi_scanner.base_key_offset, + is_restart, + self.fixed_key_len, + self.fixed_value_len, + ) + } +} + +impl, Parsed: ParsedItem> Iterator for Decoder<'_, Item, Parsed> { + type Item = Parsed; + + fn next(&mut self) -> Option { + if self.hi_scanner.base_key_offset.is_some() + && self.lo_scanner.offset >= self.hi_scanner.offset + { + return None; + } + + let is_restart: bool = self.lo_scanner.remaining_in_interval == 0; + + #[expect( + unsafe_code, + reason = "cursor is advanced by read_ operations which check for EOF, and the cursor starts at 0 - the slice is never empty" + )] + let mut reader = + Cursor::new(unsafe { self.block.data.get_unchecked(self.lo_scanner.offset..) }); + + #[expect( + clippy::cast_possible_truncation, + reason = "blocks do not even come close to 4 GiB in size" + )] + let item = Self::parse_current_item( + &mut reader, + self.lo_scanner.offset, + self.lo_scanner.base_key_offset, + is_restart, + self.fixed_key_len, + self.fixed_value_len, + ) + .inspect(|item| { + self.lo_scanner.offset += reader.position() as usize; + + if is_restart { + self.lo_scanner.base_key_offset = Some(item.key_offset()); + } + }); + + if is_restart { + self.lo_scanner.remaining_in_interval = usize::from(self.restart_interval) - 1; + } else { + self.lo_scanner.remaining_in_interval -= 1; + } + + item + } +} + +impl, Parsed: ParsedItem> DoubleEndedIterator + for Decoder<'_, Item, Parsed> +{ + fn next_back(&mut self) -> Option { + if let Some(top) = self.consume_stack_top() { + return Some(top); + } + + // NOTE: If we wrapped, we are at the end + // This is safe to do, because there cannot be that many restart intervals + if self.hi_scanner.ptr_idx == usize::MAX { + return None; + } + + self.hi_scanner.ptr_idx = self.hi_scanner.ptr_idx.wrapping_sub(1); + + // NOTE: If we wrapped, we are at the end + // This is safe to do, because there cannot be that many restart intervals + if self.hi_scanner.ptr_idx == usize::MAX { + return None; + } + + self.fill_stack(); + + self.consume_stack_top() + } +} diff --git a/crates/lsm-tree/src/table/block/encoder.rs b/crates/lsm-tree/src/table/block/encoder.rs new file mode 100644 index 000000000..89d65174c --- /dev/null +++ b/crates/lsm-tree/src/table/block/encoder.rs @@ -0,0 +1,197 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{ + super::{ + block::binary_index::Builder as BinaryIndexBuilder, + block::hash_index::{Builder as HashIndexBuilder, MAX_POINTERS_FOR_HASH_INDEX}, + util::longest_shared_prefix_length, + }, + Trailer, +}; +use std::marker::PhantomData; + +pub trait Encodable { + fn key(&self) -> &[u8]; + + fn encode_full_into( + &self, + writer: &mut W, + state: &mut Context, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> crate::Result<()> + where + Self: Sized; + + fn encode_truncated_into( + &self, + writer: &mut W, + state: &mut Context, + shared_len: usize, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> crate::Result<()> + where + Self: Sized; +} + +/// Block encoder +/// +/// The block encoder accepts an ascending stream of items, encodes them into +/// restart intervals and builds a binary seek index (and optionally a hash index). +/// +/// # Example +/// +/// A block with `restart_interval=4` +/// +/// ```js +/// _______________ +/// __________|__________ | +/// v v | | +/// [h][t][t][t][h][t][t][t][h][t][t][t][h][t][t][t][h][t][t][t][0,1,2,3,4][0 C F F 3][ptr][ptr] +/// ^ ^ ^ ^ ^ ^ ^ +/// 0 1 2 3 4 bin index hash index +/// +/// h = restart head +/// t = truncated item +/// ``` +/// +/// The binary index holds pointers to all restart heads. +/// Because restart heads hold a full key, they can be used to compare to a needle key. +/// +/// For explanation of hash index, see `hash_index/mod.rs`. +pub struct Encoder<'a, Context: Default, Item: Encodable> { + pub(crate) phantom: PhantomData<(Context, Item)>, + + pub(crate) writer: &'a mut Vec, + + pub(crate) state: Context, + + pub(crate) item_count: usize, + pub(crate) restart_count: usize, + + pub(crate) restart_interval: u8, + // pub(crate) use_prefix_truncation: bool, // TODO: support non-prefix truncation? + pub(crate) binary_index_builder: BinaryIndexBuilder, + pub(crate) hash_index_builder: HashIndexBuilder, + + pub(crate) fixed_key_len: Option, + pub(crate) fixed_value_len: Option, + + base_key: &'a [u8], +} + +// TODO: support no binary index -> use in meta blocks with restart interval = 1 +// TODO: adjust test + fuzz tests to also test for no binary index +// TODO: https://github.com/fjall-rs/lsm-tree/issues/185 + +impl<'a, Context: Default, Item: Encodable> Encoder<'a, Context, Item> { + pub fn new( + writer: &'a mut Vec, + item_count: usize, + restart_interval: u8, // TODO: should be NonZero + hash_index_ratio: f32, + first_key: &'a [u8], + ) -> Self { + let binary_index_builder = BinaryIndexBuilder::new(item_count / restart_interval as usize); + let hash_index_builder = HashIndexBuilder::with_hash_ratio(item_count, hash_index_ratio); + + Self { + phantom: PhantomData, + + writer, + + state: Context::default(), + + item_count: 0, + restart_count: 0, + + restart_interval, + // use_prefix_truncation: true, + binary_index_builder, + hash_index_builder, + + fixed_key_len: None, + fixed_value_len: None, + + base_key: first_key, + } + } + + #[must_use] + pub fn use_fixed_key_len(mut self, len: Option) -> Self { + self.fixed_key_len = len; + self + } + + #[must_use] + pub fn use_fixed_value_len(mut self, len: Option) -> Self { + self.fixed_value_len = len; + self + } + + // /// Toggles prefix truncation. + // pub fn use_prefix_truncation(mut self, flag: bool) -> Self { + // assert!(flag, "prefix truncation is currently required to be true"); + + // self.use_prefix_truncation = flag; + + // self + // } + + pub fn write(&mut self, item: &'a Item) -> crate::Result<()> { + // NOTE: Check if we are a restart marker + if self + .item_count + .is_multiple_of(usize::from(self.restart_interval)) + { + self.restart_count += 1; + + if self.restart_interval > 0 { + #[expect( + clippy::cast_possible_truncation, + reason = "we consider the caller to be trustworthy" + )] + self.binary_index_builder.insert(self.writer.len() as u32); + } + + item.encode_full_into( + &mut *self.writer, + &mut self.state, + self.fixed_key_len, + self.fixed_value_len, + )?; + + self.base_key = item.key(); + } else { + let shared_prefix_len = longest_shared_prefix_length(self.base_key, item.key()); + item.encode_truncated_into( + &mut *self.writer, + &mut self.state, + shared_prefix_len, + self.fixed_key_len, + self.fixed_value_len, + )?; + } + + let restart_idx = self.restart_count - 1; + + if self.hash_index_builder.bucket_count() > 0 && restart_idx < MAX_POINTERS_FOR_HASH_INDEX { + #[expect( + clippy::cast_possible_truncation, + reason = "max binary index is bound to u8 by MAX_POINTERS_FOR_HASH_INDEX" + )] + self.hash_index_builder.set(item.key(), restart_idx as u8); + } + + self.item_count += 1; + + Ok(()) + } + + pub fn finish(self) -> crate::Result<()> { + Trailer::write(self) + } +} diff --git a/crates/lsm-tree/src/table/block/hash_index/builder.rs b/crates/lsm-tree/src/table/block/hash_index/builder.rs new file mode 100644 index 000000000..09aa54465 --- /dev/null +++ b/crates/lsm-tree/src/table/block/hash_index/builder.rs @@ -0,0 +1,124 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{MARKER_CONFLICT, MARKER_FREE, calculate_bucket_position}; +use byteorder::WriteBytesExt; + +/// With 254, pointers [0 - 253] can be indexed. +pub const MAX_POINTERS_FOR_HASH_INDEX: usize = 254; + +/// Builds a block hash index +#[derive(Debug)] +pub struct Builder(Vec); + +impl Builder { + /// Initializes a new builder with the given number of buckets. + #[must_use] + pub fn with_bucket_count(bucket_count: u32) -> Self { + Self(vec![MARKER_FREE; bucket_count as usize]) + } + + #[must_use] + pub fn with_hash_ratio(item_count: usize, hash_ratio: f32) -> Self { + Self::with_bucket_count(Self::calculate_bucket_count(item_count, hash_ratio)) + } + + fn calculate_bucket_count(item_count: usize, hash_ratio: f32) -> u32 { + assert!( + hash_ratio.is_sign_positive(), + "hash_ratio may not be negative", + ); + + if hash_ratio > 0.0 { + #[expect( + clippy::cast_precision_loss, + reason = "item count tends to be some thousands at most, so f32 should be precise enough" + )] + let item_count = item_count as f32; + + #[expect( + clippy::cast_sign_loss, + clippy::cast_possible_truncation, + reason = "buckets tend to be in the range of a couple of hundreds to thousands, so easily fits into u32" + )] + let bucket_count = (item_count * hash_ratio) as u32; + + 1.max(bucket_count) + } else { + 0 + } + } + + /// Returns the number of buckets. + #[must_use] + #[expect( + clippy::cast_possible_truncation, + reason = "we know the hash index has a bucket count <= u8" + )] + pub fn bucket_count(&self) -> u32 { + self.0.len() as u32 + } + + /// Tries to map the given key to the binary index position. + pub fn set(&mut self, key: &[u8], binary_index_pos: u8) -> bool { + debug_assert!( + binary_index_pos <= 253, + "restart index too high for hash index" + ); + + assert!(self.bucket_count() > 0, "no buckets to insert into"); + + let bucket_pos = calculate_bucket_position(key, self.bucket_count()); + + // SAFETY: We use modulo in `calculate_bucket_position` + #[expect(unsafe_code, reason = "see safety")] + let curr_marker = unsafe { *self.0.get_unchecked(bucket_pos) }; + + match curr_marker { + MARKER_CONFLICT => false, + MARKER_FREE => { + // SAFETY: We previously asserted that the slot exists + #[expect(unsafe_code, reason = "see safety")] + unsafe { + *self.0.get_unchecked_mut(bucket_pos) = binary_index_pos; + } + + true + } + x if x == binary_index_pos => { + // If different keys map to the same bucket, we can keep + // the mapping + true + } + _ => { + // Mark as conflicted + + // SAFETY: We previously asserted that the slot exists + #[expect(unsafe_code, reason = "see safety")] + unsafe { + *self.0.get_unchecked_mut(bucket_pos) = MARKER_CONFLICT; + } + + false + } + } + } + + /// Consumes the builder, returning its raw bytes. + /// + /// Only used for tests/benchmarks + #[must_use] + #[doc(hidden)] + pub fn into_inner(self) -> Vec { + self.0 + } + + /// Appends the raw index bytes to a writer. + pub fn write(self, writer: &mut W) -> std::io::Result<()> { + for byte in self.0 { + writer.write_u8(byte)?; + } + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/block/hash_index/mod.rs b/crates/lsm-tree/src/table/block/hash_index/mod.rs new file mode 100644 index 000000000..0e49816ac --- /dev/null +++ b/crates/lsm-tree/src/table/block/hash_index/mod.rs @@ -0,0 +1,143 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +//! The hash index is a compact (typically <=1 byte per KV) index +//! embeddeded into a block to speed up point reads. +//! +//! The index is initialized with `hash_ratio * item_count` buckets. +//! +//! Each bucket is initialized as 254 (`FREE`). +//! +//! During block building, each key is hashed into a bucket. +//! If the bucket is FREE, it is set to the index of the binary index pointer +//! pointing to the item's restart interval. +//! +//! If the given bucket is already non-`FREE`, it is set to `CONFLICT`. +//! +//! During a point read, `CONFLICT`ed buckets are skipped, and the binary index +//! is consulted instead. + +mod builder; +mod reader; + +pub use builder::{Builder, MAX_POINTERS_FOR_HASH_INDEX}; +pub use reader::Reader; + +pub(crate) const MARKER_FREE: u8 = u8::MAX - 1; // 254 +pub(crate) const MARKER_CONFLICT: u8 = u8::MAX; // 255 + +/// Calculates the bucket index for the given key. +#[expect( + clippy::cast_possible_truncation, + reason = "the hash index has a bucket count <= u32" +)] +fn calculate_bucket_position(key: &[u8], bucket_count: u32) -> usize { + use crate::hash::hash64; + + let hash = hash64(key); + + (hash % u64::from(bucket_count)) as usize +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn hash_index_build_simple() { + let mut hash_index = Builder::with_bucket_count(100); + + hash_index.set(b"a", 5); + hash_index.set(b"b", 8); + hash_index.set(b"c", 10); + + let bytes = hash_index.into_inner(); + + // NOTE: Hash index bytes need to be consistent across machines and compilations etc. + assert_eq!( + [ + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 10, 254, 254, 254, 8, 254, + 254, 254, 5, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254 + ], + &*bytes + ); + + let reader = Reader::new(&bytes, 0, 100); + assert_eq!(0, reader.conflict_count()); + + assert_eq!(5, reader.get(b"a")); + assert_eq!(8, reader.get(b"b")); + assert_eq!(10, reader.get(b"c")); + assert_eq!(MARKER_FREE, reader.get(b"d")); + } + + #[test] + fn hash_index_build_conflict() { + let mut hash_index = Builder::with_bucket_count(1); + + hash_index.set(b"a", 5); + hash_index.set(b"b", 8); + + let bytes = hash_index.into_inner(); + + assert_eq!([255], &*bytes); + + assert_eq!(1, Reader::new(&bytes, 0, 1).conflict_count()); + } + + #[test] + fn hash_index_build_same_offset() { + let mut hash_index = Builder::with_bucket_count(1); + + hash_index.set(b"a", 5); + hash_index.set(b"b", 5); + + let bytes = hash_index.into_inner(); + + assert_eq!([5], &*bytes); + + let reader = Reader::new(&bytes, 0, 1); + assert_eq!(0, reader.conflict_count()); + assert_eq!(5, reader.get(b"a")); + assert_eq!(5, reader.get(b"b")); + } + + #[test] + fn hash_index_build_mix() { + let mut hash_index = Builder::with_bucket_count(1); + + hash_index.set(b"a", 5); + hash_index.set(b"b", 5); + hash_index.set(b"c", 6); + + let bytes = hash_index.into_inner(); + + assert_eq!([255], &*bytes); + + assert_eq!(1, Reader::new(&bytes, 0, 1).conflict_count()); + } + + #[test] + fn hash_index_read_conflict() { + let mut hash_index = Builder::with_bucket_count(1); + + hash_index.set(b"a", 5); + hash_index.set(b"b", 8); + + let bytes = hash_index.into_inner(); + + let reader = Reader::new(&bytes, 0, 1); + assert_eq!(MARKER_CONFLICT, reader.get(b"a")); + assert_eq!(MARKER_CONFLICT, reader.get(b"b")); + assert_eq!(MARKER_CONFLICT, reader.get(b"c")); + + assert_eq!(1, Reader::new(&bytes, 0, 1).conflict_count()); + } +} diff --git a/crates/lsm-tree/src/table/block/hash_index/reader.rs b/crates/lsm-tree/src/table/block/hash_index/reader.rs new file mode 100644 index 000000000..7b36ffbcd --- /dev/null +++ b/crates/lsm-tree/src/table/block/hash_index/reader.rs @@ -0,0 +1,59 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{MARKER_CONFLICT, calculate_bucket_position}; + +/// Helper to read from an embedded block hash index +pub struct Reader<'a>(&'a [u8]); + +impl<'a> Reader<'a> { + /// Initializes a new hash index reader. + #[must_use] + pub fn new(bytes: &'a [u8], offset: u32, len: u32) -> Self { + let offset = offset as usize; + let len = len as usize; + let end = offset + len; + + #[expect( + clippy::indexing_slicing, + reason = "we consider the caller to be trustworthy" + )] + Self(&bytes[offset..end]) + } + + /// Returns the number of buckets. + #[must_use] + pub fn bucket_count(&self) -> usize { + self.0.len() + } + + /// Returns the number of conflict markers in the hash index. + #[must_use] + #[expect( + clippy::naive_bytecount, + reason = "only used in metrics, so no need to be hyper-optimized" + )] + pub fn conflict_count(&self) -> usize { + self.0 + .iter() + .filter(|&&byte| byte == MARKER_CONFLICT) + .count() + } + + /// Returns the binary index position if the key is not conflicted. + #[must_use] + pub fn get(&self, key: &[u8]) -> u8 { + #[expect( + clippy::cast_possible_truncation, + reason = "even with very high hash ratio, there won't be nearly enough items to cause us to create ~4 billion buckets" + )] + let bucket_count = self.0.len() as u32; + + let bucket_pos = calculate_bucket_position(key, bucket_count); + + // SAFETY: We use modulo in `calculate_bucket_position` + // SAFETY: Also we already did a bounds check in the constructor using indexing slicing + *unsafe { self.0.get_unchecked(bucket_pos) } + } +} diff --git a/crates/lsm-tree/src/table/block/header.rs b/crates/lsm-tree/src/table/block/header.rs new file mode 100644 index 000000000..42e34b8f1 --- /dev/null +++ b/crates/lsm-tree/src/table/block/header.rs @@ -0,0 +1,215 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::Checksum; +use crate::checksum::ChecksummedWriter; +use crate::coding::{Decode, Encode}; +use crate::file::MAGIC_BYTES; +use crate::table::block::BlockType; +use byteorder::{ReadBytesExt, WriteBytesExt}; +use std::io::{Read, Write}; + +struct ChecksummedReader { + inner: R, + hasher: xxhash_rust::xxh3::Xxh3Default, +} + +impl ChecksummedReader { + pub fn new(reader: R) -> Self { + Self { + inner: reader, + hasher: xxhash_rust::xxh3::Xxh3Default::new(), + } + } + + pub fn checksum(&self) -> Checksum { + Checksum::from_raw(self.hasher.digest128()) + } + + /// Optionally expose the inner reader if needed + pub fn into_inner(self) -> R { + self.inner + } +} + +impl std::io::Read for ChecksummedReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + + #[expect(clippy::indexing_slicing)] + self.hasher.update(&buf[..n]); + + Ok(n) + } +} + +/// Header of a disk-based block +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct Header { + pub block_type: BlockType, + + /// Checksum value to verify integrity of data + pub checksum: Checksum, + + /// On-disk size of data segment + pub data_length: u32, + + /// Uncompressed size of data segment + pub uncompressed_length: u32, +} + +impl Header { + #[must_use] + pub const fn serialized_len() -> usize { + MAGIC_BYTES.len() + // Block type + + std::mem::size_of::() + // Data checksum + + std::mem::size_of::() + // On-disk size + + std::mem::size_of::() + // Uncompressed data length + + std::mem::size_of::() + // Checksum + + std::mem::size_of::() + } +} + +impl Encode for Header { + fn encode_into(&self, mut writer: &mut W) -> Result<(), crate::Error> { + use byteorder::LE; + + let checksum = { + let mut writer = ChecksummedWriter::new(&mut writer); + + // Write header + writer.write_all(&MAGIC_BYTES)?; + + // Write block type + writer.write_u8(self.block_type.into())?; + + // Write data checksum + writer.write_u128::(self.checksum.into_u128())?; + + // Write on-disk size length + writer.write_u32::(self.data_length)?; + + // Write uncompressed data length + writer.write_u32::(self.uncompressed_length)?; + + writer.checksum() + }; + + #[expect( + clippy::cast_possible_truncation, + reason = "we purposefully only use the lower 4 bytes as checksum" + )] + // Write 4-byte checksum + writer.write_u32::(checksum.into_u128() as u32)?; + + Ok(()) + } +} + +impl Decode for Header { + fn decode_from(reader: &mut R) -> Result { + use byteorder::LE; + + let mut protected_reader = ChecksummedReader::new(reader); + + // Check header + let mut magic = [0u8; MAGIC_BYTES.len()]; + protected_reader.read_exact(&mut magic)?; + + if magic != MAGIC_BYTES { + return Err(crate::Error::InvalidHeader("Block")); + } + + // Read block type + let block_type = protected_reader.read_u8()?; + let block_type = BlockType::try_from(block_type)?; + + // Read data checksum + let checksum = protected_reader.read_u128::()?; + + // Read data length + let data_length = protected_reader.read_u32::()?; + + // Read data length + let uncompressed_length = protected_reader.read_u32::()?; + + #[expect( + clippy::cast_possible_truncation, + reason = "we purposefully only use the lower 4 bytes as checksum" + )] + // Get header checksum + let got_checksum = protected_reader.checksum().into_u128() as u32; + let got_checksum = Checksum::from_raw(u128::from(got_checksum)); + + let reader = protected_reader.into_inner(); + + // Read & check checksum + let header_checksum: u128 = reader.read_u32::()?.into(); + let header_checksum = Checksum::from_raw(header_checksum); + + if header_checksum != got_checksum { + return Err(crate::Error::ChecksumMismatch { + got: got_checksum, + expected: header_checksum, + }); + } + + Ok(Self { + block_type, + checksum: Checksum::from_raw(checksum), + data_length, + uncompressed_length, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn block_header_serde_roundtrip() -> crate::Result<()> { + let header = Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(5), + data_length: 252_356, + uncompressed_length: 124_124_124, + }; + + let bytes = header.encode_into_vec(); + + assert_eq!(bytes.len(), Header::serialized_len()); + assert_eq!(header, Header::decode_from(&mut &bytes[..])?); + + Ok(()) + } + + #[test] + #[expect(clippy::indexing_slicing)] + fn block_header_detect_corruption() { + let header = Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(5), + data_length: 252_356, + uncompressed_length: 124_124_124, + }; + + let mut bytes = header.encode_into_vec(); + bytes[5] += 1; // mutate block type enum tag + + assert!( + matches!( + Header::decode_from(&mut &bytes[..]), + Err(crate::Error::ChecksumMismatch { .. }), + ), + "did not detect header corruption", + ); + } +} diff --git a/crates/lsm-tree/src/table/block/mod.rs b/crates/lsm-tree/src/table/block/mod.rs new file mode 100644 index 000000000..cd56dbba4 --- /dev/null +++ b/crates/lsm-tree/src/table/block/mod.rs @@ -0,0 +1,231 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub(crate) mod binary_index; +pub mod decoder; +mod encoder; +pub mod hash_index; +mod header; +mod offset; +mod trailer; +mod r#type; + +pub(crate) use decoder::{Decodable, Decoder, ParsedItem}; +pub(crate) use encoder::{Encodable, Encoder}; +pub use header::Header; +pub use offset::BlockOffset; +pub(crate) use trailer::{TRAILER_START_MARKER, Trailer}; +pub use r#type::BlockType; + +use crate::{ + Checksum, CompressionType, Slice, + coding::{Decode, Encode}, + table::BlockHandle, +}; +use std::fs::File; + +/// A block on disk +/// +/// Consists of a fixed-size header and some bytes (the data/payload). +#[derive(Clone)] +pub struct Block { + pub header: Header, + pub data: Slice, +} + +impl Block { + /// Returns the uncompressed block size in bytes. + #[must_use] + pub fn size(&self) -> usize { + self.data.len() + } + + /// Encodes a block into a writer. + pub fn write_into( + mut writer: &mut W, + data: &[u8], + block_type: BlockType, + compression: CompressionType, + ) -> crate::Result
{ + let mut header = Header { + block_type, + checksum: Checksum::from_raw(0), // <-- NOTE: Is set later on + data_length: 0, // <-- NOTE: Is set later on + + #[expect(clippy::cast_possible_truncation, reason = "blocks are limited to u32")] + uncompressed_length: data.len() as u32, + }; + + let data = match compression { + CompressionType::None => data, + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => &lz4_flex::compress(data), + }; + + #[expect(clippy::cast_possible_truncation, reason = "blocks are limited to u32")] + { + header.data_length = data.len() as u32; + header.checksum = Checksum::from_raw(crate::hash::hash128(data)); + } + + header.encode_into(&mut writer)?; + writer.write_all(data)?; + + log::trace!( + "Writing block with size {}B (compressed: {}B) (excluding header of {}B)", + header.uncompressed_length, + header.data_length, + Header::serialized_len(), + ); + + Ok(header) + } + + /// Reads a block from a reader. + pub fn from_reader( + reader: &mut R, + compression: CompressionType, + ) -> crate::Result { + let header = Header::decode_from(reader)?; + let raw_data = Slice::from_reader(reader, header.data_length as usize)?; + + let checksum = Checksum::from_raw(crate::hash::hash128(&raw_data)); + + checksum.check(header.checksum).inspect_err(|_| { + log::error!( + "Checksum mismatch for , got={}, expected={}", + checksum, + header.checksum, + ); + })?; + + let data = match compression { + CompressionType::None => raw_data, + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => { + #[warn(unsafe_code)] + let mut builder = + unsafe { Slice::builder_unzeroed(header.uncompressed_length as usize) }; + + lz4_flex::decompress_into(&raw_data, &mut builder) + .map_err(|_| crate::Error::Decompress(compression))?; + + builder.freeze().into() + } + }; + + debug_assert_eq!(header.uncompressed_length, { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + { + data.len() as u32 + } + }); + + Ok(Self { header, data }) + } + + /// Reads a block from a file. + pub fn from_file( + file: &File, + handle: BlockHandle, + compression: CompressionType, + ) -> crate::Result { + let buf = crate::file::read_exact(file, *handle.offset(), handle.size() as usize)?; + + let header = Header::decode_from(&mut &buf[..])?; + + #[expect(clippy::indexing_slicing)] + let checksum = Checksum::from_raw(crate::hash::hash128(&buf[Header::serialized_len()..])); + + checksum.check(header.checksum).inspect_err(|_| { + log::error!( + "Checksum mismatch for block {handle:?}, got={}, expected={}", + checksum, + header.checksum, + ); + })?; + + let buf = match compression { + CompressionType::None => { + let value = buf.slice(Header::serialized_len()..); + + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + { + debug_assert_eq!(header.uncompressed_length, value.len() as u32); + } + + value + } + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => { + // NOTE: We know that a header always exists and data is never empty + // So the slice is fine + #[expect(clippy::indexing_slicing)] + let raw_data = &buf[Header::serialized_len()..]; + + #[warn(unsafe_code)] + let mut builder = + unsafe { Slice::builder_unzeroed(header.uncompressed_length as usize) }; + + lz4_flex::decompress_into(raw_data, &mut builder) + .map_err(|_| crate::Error::Decompress(compression))?; + + builder.freeze().into() + } + }; + + Ok(Self { header, data: buf }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + // TODO: Block::from_file roundtrips + + #[test] + fn block_roundtrip_uncompressed() -> crate::Result<()> { + let mut writer = vec![]; + + Block::write_into( + &mut writer, + b"abcdefabcdefabcdef", + BlockType::Data, + CompressionType::None, + )?; + + { + let mut reader = &writer[..]; + let block = Block::from_reader(&mut reader, CompressionType::None)?; + assert_eq!(b"abcdefabcdefabcdef", &*block.data); + } + + Ok(()) + } + #[test] + #[cfg(feature = "lz4")] + fn block_roundtrip_lz4() -> crate::Result<()> { + let mut writer = vec![]; + + Block::write_into( + &mut writer, + b"abcdefabcdefabcdef", + BlockType::Data, + CompressionType::Lz4, + )?; + + { + let mut reader = &writer[..]; + let block = Block::from_reader(&mut reader, CompressionType::Lz4)?; + assert_eq!(b"abcdefabcdefabcdef", &*block.data); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/block/offset.rs b/crates/lsm-tree/src/table/block/offset.rs new file mode 100644 index 000000000..db7b80e81 --- /dev/null +++ b/crates/lsm-tree/src/table/block/offset.rs @@ -0,0 +1,27 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +// TODO: rename FileOffset? +#[derive(Copy, Clone, Default, Debug, std::hash::Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct BlockOffset(pub u64); + +impl std::ops::Deref for BlockOffset { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::AddAssign for BlockOffset { + fn add_assign(&mut self, rhs: Self) { + *self += *rhs; + } +} + +impl std::ops::AddAssign for BlockOffset { + fn add_assign(&mut self, rhs: u64) { + self.0 += rhs; + } +} diff --git a/crates/lsm-tree/src/table/block/trailer.rs b/crates/lsm-tree/src/table/block/trailer.rs new file mode 100644 index 000000000..769d45d9b --- /dev/null +++ b/crates/lsm-tree/src/table/block/trailer.rs @@ -0,0 +1,198 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{ + Block, + encoder::{Encodable, Encoder}, +}; +use crate::table::block::hash_index::MAX_POINTERS_FOR_HASH_INDEX; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +pub const TRAILER_START_MARKER: u8 = 255; + +const TRAILER_SIZE: usize = 5 * std::mem::size_of::() + + (2 * std::mem::size_of::()) + // Fixed key size + + std::mem::size_of::() + + std::mem::size_of::() + // Prefix truncation on/off (always on) + + std::mem::size_of::() + // Fixed value size + + std::mem::size_of::() + + std::mem::size_of::(); + +/// Block trailer +/// +/// ## Format +/// +/// \[restart_interval\] \[binary index\] \[hash index\] \[prefix truncation\] +/// \[fixed key length\] \[fixed value length\] \[item count\] +#[expect(clippy::doc_markdown)] +pub struct Trailer<'a> { + block: &'a Block, +} + +impl<'a> Trailer<'a> { + pub fn new(block: &'a Block) -> Self { + Self { block } + } + + /// Returns the trailer position. + pub fn trailer_offset(&self) -> usize { + self.block.data.len() - TRAILER_SIZE + } + + /// Returns the number of items in the block + #[must_use] + pub fn item_count(&self) -> usize { + let reader = self.as_slice(); + + #[expect( + clippy::indexing_slicing, + reason = "the item_count is at the end and is a u32" + )] + let reader = &mut &reader[(TRAILER_SIZE - std::mem::size_of::())..]; + + #[expect( + clippy::expect_used, + reason = "the trailer offset is valid, and the trailer has a fixed size so the next item must be the item count" + )] + { + reader + .read_u32::() + .expect("should read item count") as usize + } + } + + pub fn as_slice(&self) -> &[u8] { + let start = self.trailer_offset(); + + // SAFETY: We know that a block always has a trailer, so the + // `block_size - TRAILER_SIZE` cannot go out of bounds + #[expect(unsafe_code, reason = "see safety")] + unsafe { + self.block.data.get_unchecked(start..) + } + } + + pub fn write>(mut encoder: Encoder<'_, S, T>) -> crate::Result<()> { + // IMPORTANT: Terminator marker + encoder.writer.write_u8(TRAILER_START_MARKER)?; + + #[expect( + clippy::cast_possible_truncation, + reason = "data blocks will never even approach 4 GB in size" + )] + let binary_index_offset = encoder.writer.len() as u32; + + // Write binary index + let (binary_index_step_size, binary_index_len) = + encoder.binary_index_builder.write(&mut encoder.writer)?; + + let mut hash_index_offset = 0u32; + let hash_index_len = encoder.hash_index_builder.bucket_count(); + + // NOTE: We can only use a hash index when there are 254 buckets or less + // Because 254 and 255 are reserved marker values + // + // With the default restart interval of 16, that still gives us support + // for up to ~4000 KVs + if encoder.hash_index_builder.bucket_count() > 0 + && binary_index_len <= MAX_POINTERS_FOR_HASH_INDEX + { + // NOTE: We know that data blocks will never even approach 4 GB in size + #[expect(clippy::cast_possible_truncation)] + { + hash_index_offset = encoder.writer.len() as u32; + } + + // Write hash index + encoder.hash_index_builder.write(&mut encoder.writer)?; + } + + // Write trailer + + #[cfg(debug_assertions)] + let bytes_before = encoder.writer.len(); + + encoder.writer.write_u8(encoder.restart_interval)?; + + encoder.writer.write_u8(binary_index_step_size)?; + + #[expect( + clippy::cast_possible_truncation, + reason = "even with a dense index, there can't be more index pointers than items" + )] + encoder + .writer + .write_u32::(binary_index_len as u32)?; + + encoder + .writer + .write_u32::(binary_index_offset)?; + + encoder + .writer + .write_u32::(if hash_index_offset > 0 { + hash_index_len + } else { + 0 + })?; + + encoder + .writer + .write_u32::(hash_index_offset)?; + + // Prefix truncation on/off (always on) + encoder.writer.write_u8(1)?; + + // Fixed key size + encoder + .writer + .write_u8(u8::from(encoder.fixed_key_len.is_some()))?; + encoder + .writer + .write_u16::(encoder.fixed_key_len.unwrap_or_default())?; + + // Fixed value size + encoder + .writer + .write_u8(u8::from(encoder.fixed_value_len.is_some()))?; + encoder + .writer + .write_u32::(encoder.fixed_value_len.unwrap_or_default())?; + + #[expect( + clippy::cast_possible_truncation, + reason = "data blocks will never even approach 4 GiB in size, so there can't be that many items either" + )] + encoder + .writer + .write_u32::(encoder.item_count as u32)?; + + #[cfg(debug_assertions)] + assert_eq!( + TRAILER_SIZE, + encoder.writer.len() - bytes_before, + "trailer size does not match", + ); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::table::block::TRAILER_START_MARKER; + use strum::IntoEnumIterator; + use test_log::test; + + #[test] + fn value_type_never_block_trailer_start_marker() { + for variant in crate::ValueType::iter() { + let n: u8 = variant.into(); + assert_ne!(n, TRAILER_START_MARKER); + } + } +} diff --git a/crates/lsm-tree/src/table/block/type.rs b/crates/lsm-tree/src/table/block/type.rs new file mode 100644 index 000000000..82eadf11b --- /dev/null +++ b/crates/lsm-tree/src/table/block/type.rs @@ -0,0 +1,36 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum BlockType { + Data, + Index, + Filter, + Meta, +} + +impl From for u8 { + fn from(val: BlockType) -> Self { + match val { + BlockType::Data => 0, + BlockType::Index => 1, + BlockType::Filter => 2, + BlockType::Meta => 3, + } + } +} + +impl TryFrom for BlockType { + type Error = crate::Error; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Data), + 1 => Ok(Self::Index), + 2 => Ok(Self::Filter), + 3 => Ok(Self::Meta), + _ => Err(crate::Error::InvalidTag(("BlockType", value))), + } + } +} diff --git a/crates/lsm-tree/src/table/block_index/full.rs b/crates/lsm-tree/src/table/block_index/full.rs new file mode 100644 index 000000000..169b58e2c --- /dev/null +++ b/crates/lsm-tree/src/table/block_index/full.rs @@ -0,0 +1,61 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::SeqNo; +use crate::table::block_index::{BlockIndexIter, iter::OwnedIndexBlockIter}; +use crate::table::{IndexBlock, KeyedBlockHandle}; + +/// Index that translates item keys to data block handles +/// +/// The index is fully loaded into memory. +pub struct FullBlockIndex(IndexBlock); + +impl FullBlockIndex { + pub fn new(block: IndexBlock) -> Self { + Self(block) + } + + pub fn inner(&self) -> &IndexBlock { + &self.0 + } + + pub fn forward_reader(&self, needle: &[u8], seqno: SeqNo) -> Option { + let mut it = self.iter(); + if it.seek_lower(needle, seqno) { + Some(it) + } else { + None + } + } + + pub fn iter(&self) -> Iter { + Iter(OwnedIndexBlockIter::new(self.0.clone(), IndexBlock::iter)) + } +} + +pub struct Iter(OwnedIndexBlockIter); + +impl BlockIndexIter for Iter { + fn seek_lower(&mut self, key: &[u8], seqno: SeqNo) -> bool { + self.0.seek_lower(key, seqno) + } + + fn seek_upper(&mut self, key: &[u8], seqno: SeqNo) -> bool { + self.0.seek_upper(key, seqno) + } +} + +impl Iterator for Iter { + type Item = crate::Result; + + fn next(&mut self) -> Option { + self.0.next().map(Ok) + } +} + +impl DoubleEndedIterator for Iter { + fn next_back(&mut self) -> Option { + self.0.next_back().map(Ok) + } +} diff --git a/crates/lsm-tree/src/table/block_index/iter.rs b/crates/lsm-tree/src/table/block_index/iter.rs new file mode 100644 index 000000000..6660fe1c0 --- /dev/null +++ b/crates/lsm-tree/src/table/block_index/iter.rs @@ -0,0 +1,47 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + SeqNo, + table::{IndexBlock, KeyedBlockHandle, block::ParsedItem, index_block::Iter as IndexBlockIter}, +}; +use self_cell::self_cell; + +self_cell!( + pub struct OwnedIndexBlockIter { + owner: IndexBlock, + + #[covariant] + dependent: IndexBlockIter, + } +); + +impl OwnedIndexBlockIter { + pub fn seek_lower(&mut self, needle: &[u8], seqno: SeqNo) -> bool { + self.with_dependent_mut(|_, m| m.seek(needle, seqno)) + } + + pub fn seek_upper(&mut self, needle: &[u8], seqno: SeqNo) -> bool { + self.with_dependent_mut(|_, m| m.seek_upper(needle, seqno)) + } +} + +impl Iterator for OwnedIndexBlockIter { + type Item = KeyedBlockHandle; + + fn next(&mut self) -> Option { + self.with_dependent_mut(|block, iter| { + iter.next().map(|item| item.materialize(&block.inner.data)) + }) + } +} + +impl DoubleEndedIterator for OwnedIndexBlockIter { + fn next_back(&mut self) -> Option { + self.with_dependent_mut(|block, iter| { + iter.next_back() + .map(|item| item.materialize(&block.inner.data)) + }) + } +} diff --git a/crates/lsm-tree/src/table/block_index/mod.rs b/crates/lsm-tree/src/table/block_index/mod.rs new file mode 100644 index 000000000..4eb54b417 --- /dev/null +++ b/crates/lsm-tree/src/table/block_index/mod.rs @@ -0,0 +1,131 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod full; +pub mod iter; +mod two_level; +mod volatile; + +pub use full::FullBlockIndex; +pub use two_level::TwoLevelBlockIndex; +pub use volatile::VolatileBlockIndex; + +use super::KeyedBlockHandle; +use crate::SeqNo; + +pub trait BlockIndex { + fn forward_reader(&self, needle: &[u8], seqno: SeqNo) -> Option; + fn iter(&self) -> BlockIndexIterImpl; +} + +pub trait BlockIndexIter: DoubleEndedIterator> { + fn seek_lower(&mut self, key: &[u8], seqno: SeqNo) -> bool; + fn seek_upper(&mut self, key: &[u8], seqno: SeqNo) -> bool; +} + +pub enum BlockIndexIterImpl { + Full(self::full::Iter), + Volatile(self::volatile::Iter), + TwoLevel(self::two_level::Iter), +} + +impl BlockIndexIter for BlockIndexIterImpl { + fn seek_lower(&mut self, key: &[u8], seqno: SeqNo) -> bool { + match self { + Self::Full(i) => i.seek_lower(key, seqno), + Self::Volatile(i) => i.seek_lower(key, seqno), + Self::TwoLevel(i) => i.seek_lower(key, seqno), + } + } + + fn seek_upper(&mut self, key: &[u8], seqno: SeqNo) -> bool { + match self { + Self::Full(i) => i.seek_upper(key, seqno), + Self::Volatile(i) => i.seek_upper(key, seqno), + Self::TwoLevel(i) => i.seek_upper(key, seqno), + } + } +} + +impl Iterator for BlockIndexIterImpl { + type Item = crate::Result; + + fn next(&mut self) -> Option { + match self { + Self::Full(i) => i.next(), + Self::Volatile(i) => i.next(), + Self::TwoLevel(i) => i.next(), + } + } +} + +impl DoubleEndedIterator for BlockIndexIterImpl { + fn next_back(&mut self) -> Option<::Item> { + match self { + Self::Full(i) => i.next_back(), + Self::Volatile(i) => i.next_back(), + Self::TwoLevel(i) => i.next_back(), + } + } +} + +/// The block index stores references to the positions of blocks on a file and their size +/// +/// __________________ +/// | | +/// | BLOCK0 | +/// |________________| <- 'G': 0x0 +/// | | +/// | BLOCK1 | +/// |________________| <- 'M': 0x... +/// | | +/// | BLOCK2 | +/// |________________| <- 'Z': 0x... +/// +/// The block information can be accessed by key. +/// Because the blocks are sorted, any entries not covered by the index (it is sparse) can be +/// found by finding the highest block that has a lower or equal end key than the searched key (by performing in-memory binary search). +/// In the diagram above, searching for 'J' yields the block starting with 'G'. +/// 'J' must be in that block, because the next block starts with 'M'). +pub enum BlockIndexImpl { + Full(FullBlockIndex), + VolatileFull(VolatileBlockIndex), + TwoLevel(TwoLevelBlockIndex), +} + +impl BlockIndex for BlockIndexImpl { + fn forward_reader(&self, needle: &[u8], seqno: SeqNo) -> Option { + match self { + Self::Full(index) => index + .forward_reader(needle, seqno) + .map(BlockIndexIterImpl::Full), + Self::VolatileFull(index) => { + let mut it = index.iter(); + + if it.seek_lower(needle, seqno) { + Some(BlockIndexIterImpl::Volatile(it)) + } else { + None + } + } + Self::TwoLevel(index) => { + let mut it = index.iter(); + + if it.seek_lower(needle, seqno) { + Some(BlockIndexIterImpl::TwoLevel(it)) + } else { + None + } + } + } + } + + fn iter(&self) -> BlockIndexIterImpl { + match self { + Self::Full(index) => BlockIndexIterImpl::Full(index.iter()), + Self::VolatileFull(index) => BlockIndexIterImpl::Volatile(index.iter()), + Self::TwoLevel(index) => BlockIndexIterImpl::TwoLevel(index.iter()), + } + } +} diff --git a/crates/lsm-tree/src/table/block_index/two_level.rs b/crates/lsm-tree/src/table/block_index/two_level.rs new file mode 100644 index 000000000..3d0e3b5fe --- /dev/null +++ b/crates/lsm-tree/src/table/block_index/two_level.rs @@ -0,0 +1,236 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::SeqNo; +use crate::file_accessor::FileAccessor; +use crate::table::{IndexBlock, KeyedBlockHandle}; +use crate::{ + Cache, CompressionType, GlobalTableId, UserKey, + table::{ + block::BlockType, + block_index::{BlockIndexIter, iter::OwnedIndexBlockIter}, + util::load_block, + }, +}; +use std::{path::PathBuf, sync::Arc}; + +#[cfg(feature = "metrics")] +use crate::Metrics; + +/// Index that translates item keys to data block handles +/// +/// Only the top-level index is loaded into memory. +pub struct TwoLevelBlockIndex { + pub(crate) top_level_index: IndexBlock, + pub(crate) table_id: GlobalTableId, + pub(crate) path: Arc, + pub(crate) file_accessor: FileAccessor, + pub(crate) cache: Arc, + pub(crate) compression: CompressionType, + + #[cfg(feature = "metrics")] + pub(crate) metrics: Arc, +} + +impl TwoLevelBlockIndex { + pub fn iter(&self) -> Iter { + Iter { + tli_block: self.top_level_index.clone(), + tli: None, + lo_consumer: None, + hi_consumer: None, + lo: None, + hi: None, + table_id: self.table_id, + path: self.path.clone(), + file_accessor: self.file_accessor.clone(), + cache: self.cache.clone(), + compression: self.compression, + + #[cfg(feature = "metrics")] + metrics: self.metrics.clone(), + } + } +} + +pub struct Iter { + tli_block: IndexBlock, + tli: Option, + + lo_consumer: Option, + hi_consumer: Option, + + lo: Option<(UserKey, SeqNo)>, + hi: Option<(UserKey, SeqNo)>, + + table_id: GlobalTableId, + path: Arc, + file_accessor: FileAccessor, + cache: Arc, + compression: CompressionType, + + #[cfg(feature = "metrics")] + metrics: Arc, +} + +impl Iter { + fn init_tli(&mut self) -> bool { + let mut iter = OwnedIndexBlockIter::new(self.tli_block.clone(), IndexBlock::iter); + + if let Some((lo_key, lo_seqno)) = &self.lo { + if !iter.seek_lower(lo_key, *lo_seqno) { + return false; + } + } + if let Some((hi_key, hi_seqno)) = &self.hi { + if !iter.seek_upper(hi_key, *hi_seqno) { + return false; + } + } + + self.tli = Some(iter); + + true + } +} + +impl BlockIndexIter for Iter { + fn seek_lower(&mut self, key: &[u8], seqno: SeqNo) -> bool { + self.lo = Some((key.into(), seqno)); + true + } + + fn seek_upper(&mut self, key: &[u8], seqno: SeqNo) -> bool { + self.hi = Some((key.into(), seqno)); + true + } +} + +impl Iterator for Iter { + type Item = crate::Result; + + fn next(&mut self) -> Option { + if let Some(lo_block) = &mut self.lo_consumer { + if let Some(item) = lo_block.next() { + return Some(Ok(item)); + } + } + + if self.tli.is_none() && !self.init_tli() { + return None; + } + + if let Some(tli) = &mut self.tli { + let next_lowest_block = tli.next(); + + if let Some(handle) = next_lowest_block { + let block = fail_iter!(load_block( + self.table_id, + &self.path, + &self.file_accessor, + &self.cache, + &handle.into_inner(), + BlockType::Index, + self.compression, + #[cfg(feature = "metrics")] + &self.metrics, + )); + let index_block = IndexBlock::new(block); + + let mut iter = OwnedIndexBlockIter::new(index_block, IndexBlock::iter); + + if let Some((lo_key, lo_seqno)) = &self.lo { + if !iter.seek_lower(lo_key, *lo_seqno) { + return None; + } + } + if let Some((hi_key, hi_seqno)) = &self.hi { + if !iter.seek_upper(hi_key, *hi_seqno) { + return None; + } + } + + let next_item = iter.next().map(Ok); + + self.lo_consumer = Some(iter); + + if let Some(item) = next_item { + return Some(item); + } + } + } + + // Nothing more found, consume from hi consumer + if let Some(hi_block) = &mut self.hi_consumer { + if let Some(item) = hi_block.next() { + return Some(Ok(item)); + } + } + + None + } +} + +impl DoubleEndedIterator for Iter { + fn next_back(&mut self) -> Option { + if let Some(hi_block) = &mut self.hi_consumer { + if let Some(item) = hi_block.next_back() { + return Some(Ok(item)); + } + } + + if self.tli.is_none() && !self.init_tli() { + return None; + } + + if let Some(tli) = &mut self.tli { + let next_highest_block = tli.next_back(); + + if let Some(handle) = next_highest_block { + let block = fail_iter!(load_block( + self.table_id, + &self.path, + &self.file_accessor, + &self.cache, + &handle.into_inner(), + BlockType::Index, + self.compression, + #[cfg(feature = "metrics")] + &self.metrics, + )); + let index_block = IndexBlock::new(block); + + let mut iter = OwnedIndexBlockIter::new(index_block, IndexBlock::iter); + + if let Some((lo_key, lo_seqno)) = &self.lo { + if !iter.seek_lower(lo_key, *lo_seqno) { + return None; + } + } + if let Some((hi_key, hi_seqno)) = &self.hi { + if !iter.seek_upper(hi_key, *hi_seqno) { + return None; + } + } + + let next_item = iter.next_back().map(Ok); + + self.hi_consumer = Some(iter); + + if let Some(item) = next_item { + return Some(item); + } + } + } + + // Nothing more found, consume from lo consumer + if let Some(lo_block) = &mut self.lo_consumer { + if let Some(item) = lo_block.next_back() { + return Some(Ok(item)); + } + } + + None + } +} diff --git a/crates/lsm-tree/src/table/block_index/volatile.rs b/crates/lsm-tree/src/table/block_index/volatile.rs new file mode 100644 index 000000000..ad957d3ec --- /dev/null +++ b/crates/lsm-tree/src/table/block_index/volatile.rs @@ -0,0 +1,176 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::KeyedBlockHandle; +use crate::{ + Cache, CompressionType, GlobalTableId, SeqNo, UserKey, + file_accessor::FileAccessor, + table::{ + BlockHandle, IndexBlock, + block::BlockType, + block_index::{BlockIndexIter, iter::OwnedIndexBlockIter}, + util::load_block, + }, +}; +use std::{path::PathBuf, sync::Arc}; + +#[cfg(feature = "metrics")] +use crate::Metrics; + +/// Index that translates item keys to data block handles +/// +/// The index is loaded on demand. +pub struct VolatileBlockIndex { + pub(crate) table_id: GlobalTableId, + pub(crate) path: Arc, + pub(crate) file_accessor: FileAccessor, + pub(crate) cache: Arc, + pub(crate) handle: BlockHandle, + pub(crate) compression: CompressionType, + + #[cfg(feature = "metrics")] + pub(crate) metrics: Arc, +} + +impl VolatileBlockIndex { + pub fn forward_reader(&self, needle: &[u8], seqno: SeqNo) -> Iter { + let mut iter = Iter::new(self); + iter.seek_lower(needle, seqno); + iter + } + + pub fn iter(&self) -> Iter { + Iter::new(self) + } +} + +pub struct Iter { + inner: Option, + table_id: GlobalTableId, + path: Arc, + file_accessor: FileAccessor, + cache: Arc, + handle: BlockHandle, + compression: CompressionType, + + lo: Option<(UserKey, SeqNo)>, + hi: Option<(UserKey, SeqNo)>, + + #[cfg(feature = "metrics")] + pub(crate) metrics: Arc, +} + +impl Iter { + fn new(index: &VolatileBlockIndex) -> Self { + Self { + inner: None, + table_id: index.table_id, + path: index.path.clone(), + file_accessor: index.file_accessor.clone(), + cache: index.cache.clone(), + handle: index.handle, + compression: index.compression, + + lo: None, + hi: None, + + #[cfg(feature = "metrics")] + metrics: index.metrics.clone(), + } + } +} + +impl BlockIndexIter for Iter { + fn seek_lower(&mut self, key: &[u8], seqno: SeqNo) -> bool { + self.lo = Some((key.into(), seqno)); + true + } + + fn seek_upper(&mut self, key: &[u8], seqno: SeqNo) -> bool { + self.hi = Some((key.into(), seqno)); + true + } +} + +impl Iterator for Iter { + type Item = crate::Result; + + fn next(&mut self) -> Option { + if let Some(inner) = &mut self.inner { + inner.next().map(Ok) + } else { + let block = fail_iter!(load_block( + self.table_id, + &self.path, + &self.file_accessor, + &self.cache, + &self.handle, + BlockType::Index, + self.compression, + #[cfg(feature = "metrics")] + &self.metrics, + )); + let index_block = IndexBlock::new(block); + + let mut iter = OwnedIndexBlockIter::new(index_block, IndexBlock::iter); + + if let Some((lo_key, lo_seqno)) = &self.lo { + if !iter.seek_lower(lo_key, *lo_seqno) { + return None; + } + } + if let Some((hi_key, hi_seqno)) = &self.hi { + if !iter.seek_upper(hi_key, *hi_seqno) { + return None; + } + } + + let next_item = iter.next().map(Ok); + + self.inner = Some(iter); + + next_item + } + } +} + +impl DoubleEndedIterator for Iter { + fn next_back(&mut self) -> Option { + if let Some(inner) = &mut self.inner { + inner.next_back().map(Ok) + } else { + let block = fail_iter!(load_block( + self.table_id, + &self.path, + &self.file_accessor, + &self.cache, + &self.handle, + BlockType::Index, + self.compression, + #[cfg(feature = "metrics")] + &self.metrics, + )); + let index_block = IndexBlock::new(block); + + let mut iter = OwnedIndexBlockIter::new(index_block, IndexBlock::iter); + + if let Some((lo_key, lo_seqno)) = &self.lo { + if !iter.seek_lower(lo_key, *lo_seqno) { + return None; + } + } + if let Some((hi_key, hi_seqno)) = &self.hi { + if !iter.seek_upper(hi_key, *hi_seqno) { + return None; + } + } + + let next_item = iter.next_back().map(Ok); + + self.inner = Some(iter); + + next_item + } + } +} diff --git a/crates/lsm-tree/src/table/data_block/iter.rs b/crates/lsm-tree/src/table/data_block/iter.rs new file mode 100644 index 000000000..78fc62af4 --- /dev/null +++ b/crates/lsm-tree/src/table/data_block/iter.rs @@ -0,0 +1,191 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + InternalValue, + double_ended_peekable::{DoubleEndedPeekable, DoubleEndedPeekableExt}, + table::{ + block::{Decoder, ParsedItem}, + data_block::DataBlockParsedItem, + }, +}; + +/// The data block iterator handles double-ended scans over a data block +pub struct Iter<'a> { + bytes: &'a [u8], + decoder: + DoubleEndedPeekable>, +} + +impl<'a> Iter<'a> { + /// Creates a new iterator over a data block. + #[must_use] + pub fn new(bytes: &'a [u8], decoder: Decoder<'a, InternalValue, DataBlockParsedItem>) -> Self { + let decoder = decoder.double_ended_peekable(); + Self { bytes, decoder } + } + + /// Seek the iterator to an byte offset. + /// + /// This is used when the hash index returns a hit. + pub fn seek_to_offset(&mut self, offset: usize) -> bool { + self.decoder.inner_mut().set_lo_offset(offset); + true + } + + pub fn seek(&mut self, needle: &[u8]) -> bool { + // Find the restart interval whose head key is the last one strictly below `needle`. + // The decoder then performs a linear scan within that interval; we stop as soon as we + // reach a key ≥ needle. This minimizes parsing work while preserving correctness. + if !self + .decoder + .inner_mut() + .seek(|head_key, _| head_key < needle, false) + { + return false; + } + + // TODO: make sure we only linear scan over the current restart interval + // TODO: if we do more steps, something has gone wrong with the seek probably, maybe...? + + // Linear scan + loop { + let Some(item) = self.decoder.peek() else { + return false; + }; + + match item.compare_key(needle, self.bytes) { + std::cmp::Ordering::Equal => { + return true; + } + std::cmp::Ordering::Greater => { + return false; + } + std::cmp::Ordering::Less => { + // Continue + + #[expect( + clippy::expect_used, + reason = "we peeked a value successfully, so there must be a next item in the stream" + )] + self.decoder.next().expect("should exist"); + } + } + } + } + + pub fn seek_upper(&mut self, needle: &[u8]) -> bool { + // Reverse-bound seek: position the high scanner at the first restart whose head key is + // ≤ needle, then walk backwards inside the interval until we find a key ≤ needle. + if !self + .decoder + .inner_mut() + .seek_upper(|head_key, _| head_key <= needle, false) + { + return false; + } + + // Linear scan + loop { + let Some(item) = self.decoder.peek_back() else { + return false; + }; + + match item.compare_key(needle, self.bytes) { + std::cmp::Ordering::Equal => { + return true; + } + std::cmp::Ordering::Less => { + return false; + } + std::cmp::Ordering::Greater => { + // Continue + + #[expect( + clippy::expect_used, + reason = "we peeked a value successfully, so there must be a next item in the stream" + )] + self.decoder.next_back().expect("should exist"); + } + } + } + } + + pub fn seek_exclusive(&mut self, needle: &[u8]) -> bool { + // Exclusive lower bound: identical to `seek`, except we must not yield entries equal to + // `needle`. We therefore keep consuming while keys compare equal and only stop once we + // observe a strictly greater key. + if !self + .decoder + .inner_mut() + .seek(|head_key, _| head_key < needle, false) + { + return false; + } + + loop { + let Some(item) = self.decoder.peek() else { + return false; + }; + + match item.compare_key(needle, self.bytes) { + std::cmp::Ordering::Greater => { + return true; + } + std::cmp::Ordering::Equal | std::cmp::Ordering::Less => { + #[expect( + clippy::expect_used, + reason = "we peeked a value successfully, so there must be a next item in the stream" + )] + self.decoder.next().expect("should exist"); + } + } + } + } + + pub fn seek_upper_exclusive(&mut self, needle: &[u8]) -> bool { + // Exclusive upper bound: mirror of `seek_upper`. We must not include entries equal to + // `needle`, so we consume equals from the high end until we see a strictly smaller key. + if !self + .decoder + .inner_mut() + .seek_upper(|head_key, _| head_key <= needle, false) + { + return false; + } + + loop { + let Some(item) = self.decoder.peek_back() else { + return false; + }; + + match item.compare_key(needle, self.bytes) { + std::cmp::Ordering::Less => { + return true; + } + std::cmp::Ordering::Equal | std::cmp::Ordering::Greater => { + #[expect( + clippy::expect_used, + reason = "we peeked a value successfully, so there must be a next item in the stream" + )] + self.decoder.next_back().expect("should exist"); + } + } + } + } +} + +impl Iterator for Iter<'_> { + type Item = DataBlockParsedItem; + + fn next(&mut self) -> Option { + self.decoder.next() + } +} + +impl DoubleEndedIterator for Iter<'_> { + fn next_back(&mut self) -> Option { + self.decoder.next_back() + } +} diff --git a/crates/lsm-tree/src/table/data_block/iter_test.rs b/crates/lsm-tree/src/table/data_block/iter_test.rs new file mode 100644 index 000000000..e439ef677 --- /dev/null +++ b/crates/lsm-tree/src/table/data_block/iter_test.rs @@ -0,0 +1,1280 @@ +#[expect(clippy::expect_used)] +mod tests { + use crate::{ + Checksum, InternalValue, Slice, + ValueType::{Tombstone, Value}, + table::{ + Block, DataBlock, + block::{BlockType, Header, ParsedItem}, + }, + }; + use test_log::test; + + #[test] + fn data_block_wtf() -> crate::Result<()> { + let keys = [ + [0, 0, 0, 0, 0, 0, 0, 108], + [0, 0, 0, 0, 0, 0, 0, 109], + [0, 0, 0, 0, 0, 0, 0, 110], + [0, 0, 0, 0, 0, 0, 0, 111], + [0, 0, 0, 0, 0, 0, 0, 112], + [0, 0, 0, 0, 0, 0, 0, 113], + [0, 0, 0, 0, 0, 0, 0, 114], + [0, 0, 0, 0, 0, 0, 0, 115], + [0, 0, 0, 0, 0, 0, 0, 116], + [0, 0, 0, 0, 0, 0, 0, 117], + [0, 0, 0, 0, 0, 0, 0, 118], + [0, 0, 0, 0, 0, 0, 0, 119], + [0, 0, 0, 0, 0, 0, 0, 120], + [0, 0, 0, 0, 0, 0, 0, 121], + [0, 0, 0, 0, 0, 0, 0, 122], + [0, 0, 0, 0, 0, 0, 0, 123], + [0, 0, 0, 0, 0, 0, 0, 124], + [0, 0, 0, 0, 0, 0, 0, 125], + [0, 0, 0, 0, 0, 0, 0, 126], + [0, 0, 0, 0, 0, 0, 0, 127], + [0, 0, 0, 0, 0, 0, 0, 128], + [0, 0, 0, 0, 0, 0, 0, 129], + [0, 0, 0, 0, 0, 0, 0, 130], + [0, 0, 0, 0, 0, 0, 0, 131], + [0, 0, 0, 0, 0, 0, 0, 132], + [0, 0, 0, 0, 0, 0, 0, 133], + [0, 0, 0, 0, 0, 0, 0, 134], + [0, 0, 0, 0, 0, 0, 0, 135], + [0, 0, 0, 0, 0, 0, 0, 136], + [0, 0, 0, 0, 0, 0, 0, 137], + [0, 0, 0, 0, 0, 0, 0, 138], + [0, 0, 0, 0, 0, 0, 0, 139], + [0, 0, 0, 0, 0, 0, 0, 140], + [0, 0, 0, 0, 0, 0, 0, 141], + [0, 0, 0, 0, 0, 0, 0, 142], + [0, 0, 0, 0, 0, 0, 0, 143], + ]; + + let items = keys + .into_iter() + .map(|key| InternalValue::from_components(key, "", 0, Value)) + .collect::>(); + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + { + let mut iter = data_block.iter(); + iter.seek(&10u64.to_be_bytes()); + iter.seek_upper(&110u64.to_be_bytes()); + let iter = iter.map(|x| x.materialize(data_block.as_slice())); + + assert_eq!( + items.iter().take(3).cloned().collect::>(), + iter.collect::>(), + ); + } + + { + let mut iter: crate::table::data_block::Iter<'_> = data_block.iter(); + iter.seek(&10u64.to_be_bytes()); + iter.seek_upper(&110u64.to_be_bytes()); + let iter = iter.map(|x| x.materialize(data_block.as_slice())); + + assert_eq!( + items.iter().take(3).rev().cloned().collect::>(), + iter.rev().collect::>(), + ); + } + + { + let mut iter = data_block.iter(); + iter.seek(&10u64.to_be_bytes()); + iter.seek_upper(&110u64.to_be_bytes()); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + let mut count = 0; + + for x in 0.. { + if x % 2 == 0 { + let Some(_) = iter.next() else { + break; + }; + + count += 1; + } else { + let Some(_) = iter.next_back() else { + break; + }; + + count += 1; + } + } + + assert_eq!(3, count); + } + } + + Ok(()) + } + + #[test] + fn data_block_range() -> crate::Result<()> { + let items = (100u64..110) + .map(|i| InternalValue::from_components(i.to_be_bytes(), "", 0, Value)) + .collect::>(); + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + { + let mut iter = data_block.iter(); + iter.seek(&10u64.to_be_bytes()); + iter.seek_upper(&109u64.to_be_bytes()); + let iter = iter.map(|x| x.materialize(data_block.as_slice())); + + assert_eq!( + items.iter().take(10).cloned().collect::>(), + iter.collect::>(), + ); + } + + { + let mut iter: crate::table::data_block::Iter<'_> = data_block.iter(); + iter.seek(&10u64.to_be_bytes()); + iter.seek_upper(&109u64.to_be_bytes()); + let iter = iter.map(|x| x.materialize(data_block.as_slice())); + + assert_eq!( + items.iter().take(10).rev().cloned().collect::>(), + iter.rev().collect::>(), + ); + } + + { + let mut iter = data_block.iter(); + iter.seek(&10u64.to_be_bytes()); + iter.seek_upper(&109u64.to_be_bytes()); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + let mut count = 0; + + for x in 0.. { + if x % 2 == 0 { + let Some(_) = iter.next() else { + break; + }; + + count += 1; + } else { + let Some(_) = iter.next_back() else { + break; + }; + + count += 1; + } + } + + assert_eq!(10, count); + } + } + + Ok(()) + } + + #[test] + fn data_block_range_ping_pong() -> crate::Result<()> { + let items = (0u64..100) + .map(|i| InternalValue::from_components(i.to_be_bytes(), "", 0, Value)) + .collect::>(); + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + iter.seek(&5u64.to_be_bytes()); + iter.seek_upper(&9u64.to_be_bytes()); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + let mut count = 0; + + for x in 0.. { + if x % 2 == 0 { + let Some(_) = iter.next() else { + break; + }; + + count += 1; + } else { + let Some(_) = iter.next_back() else { + break; + }; + + count += 1; + } + } + + assert_eq!(5, count); + } + + Ok(()) + } + + #[test] + fn data_block_iter_forward() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!(items, &*real_items); + } + + Ok(()) + } + + #[test] + fn data_block_iter_rev() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let iter = data_block + .iter() + .rev() + .map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().rev().cloned().collect::>(), + &*real_items, + ); + } + + Ok(()) + } + + #[test] + fn data_block_iter_rev_seek_back() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"d"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.rev().collect(); + + assert_eq!( + items.iter().rev().skip(2).cloned().collect::>(), + &*real_items, + ); + } + + Ok(()) + } + + #[test] + fn data_block_iter_range_edges() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + { + let mut iter = data_block.iter(); + + assert!(!iter.seek(b"a"), "should not seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!(items.to_vec(), &*real_items); + } + + { + let mut iter = data_block.iter(); + + assert!(!iter.seek_upper(b"g"), "should not seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!(items.to_vec(), &*real_items); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"b"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().take(1).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek(b"f"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().rev().take(1).cloned().collect::>(), + &*real_items, + ); + } + } + + Ok(()) + } + + #[test] + fn data_block_iter_range() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(iter.seek(b"c"), "should seek"); + assert!(iter.seek_upper(b"d"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().skip(1).take(2).cloned().collect::>(), + &*real_items, + ); + } + + Ok(()) + } + + #[test] + fn data_block_iter_only_first() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"b"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().take(1).cloned().collect::>(), + &*real_items, + ); + } + + Ok(()) + } + + #[test] + fn data_block_iter_range_same_key() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + { + let mut iter = data_block.iter(); + + assert!(iter.seek(b"d"), "should seek"); + assert!(iter.seek_upper(b"d"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().skip(2).take(1).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"d"), "should seek"); + assert!(iter.seek(b"d"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().skip(2).take(1).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek(b"d"), "should seek"); + assert!(iter.seek_upper(b"d"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.rev().collect(); + + assert_eq!( + items + .iter() + .rev() + .skip(2) + .take(1) + .cloned() + .collect::>(), + &*real_items, + ); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"d"), "should seek"); + assert!(iter.seek(b"d"), "should seek"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.rev().collect(); + + assert_eq!( + items + .iter() + .rev() + .skip(2) + .take(1) + .cloned() + .collect::>(), + &*real_items, + ); + } + } + + Ok(()) + } + + #[test] + fn data_block_iter_range_empty() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + { + let mut iter = data_block.iter(); + + assert!(iter.seek(b"f"), "should seek"); + iter.seek_upper(b"e"); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + assert!(iter.next().is_none(), "iter should be empty"); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek(b"f"), "should seek"); + iter.seek_upper(b"e"); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + assert!(iter.next_back().is_none(), "iter should be empty"); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"e"), "should seek"); + iter.seek(b"f"); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + assert!(iter.next_back().is_none(), "iter should be empty"); + } + + { + let mut iter = data_block.iter(); + + assert!(iter.seek_upper(b"e"), "should seek"); + iter.seek(b"f"); + + let mut iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + assert!(iter.next_back().is_none(), "iter should be empty"); + } + } + + Ok(()) + } + + #[test] + fn data_block_iter_forward_seek_restart_head() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(iter.seek(b"b"), "should seek correctly"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!(items, &*real_items); + } + + Ok(()) + } + + #[test] + fn data_block_iter_forward_seek_in_interval() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(iter.seek(b"d"), "should seek correctly"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().skip(2).cloned().collect::>(), + real_items, + ); + } + + Ok(()) + } + + #[test] + fn data_block_iter_forward_seek_last() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(iter.seek(b"f"), "should seek correctly"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!( + items.iter().skip(4).cloned().collect::>(), + real_items, + ); + } + + Ok(()) + } + + #[test] + fn data_block_iter_forward_seek_before_first() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(!iter.seek(b"a"), "should not find exact match"); + + let iter = iter.map(|item| item.materialize(&data_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!(items, &*real_items); + } + + Ok(()) + } + + #[test] + fn data_block_iter_forward_seek_after_last() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = data_block.iter(); + + assert!(!iter.seek(b"g"), "should not find exact match"); + + assert!(iter.next().is_none(), "should not collect any items"); + } + + Ok(()) + } + + #[test] + fn data_block_iter_consume_last_back() -> crate::Result<()> { + let items = [ + InternalValue::from_components("pla:earth:fact", "eaaaaaaaaarth", 0, Value), + InternalValue::from_components("pla:jupiter:fact", "Jupiter is big", 0, Value), + InternalValue::from_components("pla:jupiter:mass", "Massive", 0, Value), + InternalValue::from_components("pla:jupiter:name", "Jupiter", 0, Value), + InternalValue::from_components("pla:jupiter:radius", "Big", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!(data_block.hash_bucket_count().is_none()); + + { + let mut iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!( + b"pla:earth:fact", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:fact", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:mass", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:name", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:radius", + &*iter.next().expect("should exist").key.user_key, + ); + assert!(iter.next_back().is_none()); + assert!(iter.next().is_none()); + } + + { + let mut iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!( + b"pla:earth:fact", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:fact", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:mass", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:name", + &*iter.next().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:radius", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert!(iter.next().is_none()); + assert!(iter.next_back().is_none()); + } + } + + Ok(()) + } + + #[test] + fn data_block_iter_consume_last_forwards() -> crate::Result<()> { + let items = [ + InternalValue::from_components("pla:earth:fact", "eaaaaaaaaarth", 0, Value), + InternalValue::from_components("pla:jupiter:fact", "Jupiter is big", 0, Value), + InternalValue::from_components("pla:jupiter:mass", "Massive", 0, Value), + InternalValue::from_components("pla:jupiter:name", "Jupiter", 0, Value), + InternalValue::from_components("pla:jupiter:radius", "Big", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!(data_block.hash_bucket_count().is_none()); + + { + let mut iter = data_block + .iter() + .rev() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!( + b"pla:earth:fact", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:fact", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:mass", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:name", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:radius", + &*iter.next().expect("should exist").key.user_key, + ); + assert!(iter.next().is_none()); + assert!(iter.next_back().is_none()); + } + + { + let mut iter = data_block + .iter() + .rev() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!( + b"pla:earth:fact", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:fact", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:mass", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:name", + &*iter.next_back().expect("should exist").key.user_key, + ); + assert_eq!( + b"pla:jupiter:radius", + &*iter.next().expect("should exist").key.user_key, + ); + assert!(iter.next_back().is_none()); + assert!(iter.next().is_none()); + } + } + + Ok(()) + } + + #[test] + fn data_block_iter_ping_pong_exhaust() -> crate::Result<()> { + let items = [ + InternalValue::from_components("a", "a", 0, Value), + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 0, Value), + InternalValue::from_components("e", "e", 0, Value), + ]; + + for restart_interval in 1..=u8::MAX { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!(data_block.hash_bucket_count().is_none()); + + { + let mut iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!(b"a", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"b", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"c", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"d", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"e", &*iter.next().expect("should exist").key.user_key); + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); + } + + { + let mut iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!(b"e", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"d", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"c", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"b", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"a", &*iter.next_back().expect("should exist").key.user_key); + assert!(iter.next_back().is_none()); + assert!(iter.next_back().is_none()); + } + + { + let mut iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!(b"a", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"b", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"c", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"d", &*iter.next().expect("should exist").key.user_key); + assert_eq!(b"e", &*iter.next().expect("should exist").key.user_key); + assert!(iter.next_back().is_none()); + assert!(iter.next_back().is_none()); + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); + } + + { + let mut iter = data_block + .iter() + .map(|item| item.materialize(&data_block.inner.data)); + + assert_eq!(b"e", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"d", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"c", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"b", &*iter.next_back().expect("should exist").key.user_key); + assert_eq!(b"a", &*iter.next_back().expect("should exist").key.user_key); + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); + assert!(iter.next_back().is_none()); + assert!(iter.next_back().is_none()); + } + } + + Ok(()) + } + + #[test] + fn data_block_iter_fuzz_3() -> crate::Result<()> { + let items = [ + InternalValue::from_components( + Slice::from([ + 255, 255, 255, 255, 5, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, + ]), + Slice::from([0, 0, 192]), + 18_446_744_073_701_163_007, + Tombstone, + ), + InternalValue::from_components( + Slice::from([255, 255, 255, 255, 255, 255, 0]), + Slice::from([]), + 0, + Value, + ), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 5, 1.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!( + data_block + .hash_bucket_count() + .expect("should have built hash index") + > 0, + ); + + assert_eq!(data_block.iter().count(), items.len()); + + Ok(()) + } + + #[test] + fn data_block_iter_fuzz_4() -> crate::Result<()> { + let items = [ + InternalValue::from_components( + Slice::new(&[0]), + Slice::empty(), + 3_834_029_160_418_063_669, + Value, + ), + InternalValue::from_components(Slice::new(&[0]), Slice::new(&[]), 127, Tombstone), + InternalValue::from_components( + Slice::new(&[53, 53, 53]), + Slice::empty(), + 18_446_744_073_709_551_615, + Tombstone, + ), + InternalValue::from_components( + Slice::new(&[255]), + Slice::empty(), + 18_446_744_069_414_584_831, + Tombstone, + ), + InternalValue::from_components(Slice::new(&[255, 255]), Slice::empty(), 47, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 2, 1.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!( + data_block + .hash_bucket_count() + .expect("should have built hash index") + > 0, + ); + + assert_eq!(data_block.iter().count(), items.len()); + + Ok(()) + } + + #[test] + fn data_block_seek_closed_range() -> crate::Result<()> { + let items = [ + InternalValue::from_components(Slice::new(&[0, 161]), Slice::empty(), 1, Tombstone), + InternalValue::from_components(Slice::new(&[0, 161]), Slice::empty(), 0, Tombstone), + InternalValue::from_components(Slice::new(&[1]), Slice::empty(), 0, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 100, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert_eq!(data_block.iter().count(), items.len()); + + let mut iter = data_block.iter(); + iter.seek(&[0]); + iter.seek_upper(&[0]); + + assert_eq!(0, iter.count()); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/data_block/mod.rs b/crates/lsm-tree/src/table/data_block/mod.rs new file mode 100644 index 000000000..777f3b9dd --- /dev/null +++ b/crates/lsm-tree/src/table/data_block/mod.rs @@ -0,0 +1,1378 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod iter; + +#[cfg(test)] +mod iter_test; + +pub use iter::Iter; + +use super::block::{ + Block, Decodable, Decoder, Encodable, Encoder, ParsedItem, TRAILER_START_MARKER, Trailer, + binary_index::Reader as BinaryIndexReader, hash_index::Reader as HashIndexReader, +}; +use crate::key::InternalKey; +use crate::table::block::hash_index::{MARKER_CONFLICT, MARKER_FREE}; +use crate::table::util::{SliceIndexes, compare_prefixed_slice}; +use crate::{InternalValue, SeqNo, Slice, ValueType, value::PointReadValue}; +use byteorder::WriteBytesExt; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::io::Cursor; +use std::io::Seek; +use varint_rs::{VarintReader, VarintWriter}; + +impl Decodable for InternalValue { + fn parse_restart_key<'a>( + reader: &mut Cursor<&[u8]>, + offset: usize, + data: &'a [u8], + fixed_key_len: Option, + _fixed_value_len: Option, + ) -> Option<(&'a [u8], SeqNo)> { + let value_type = unwrap!(reader.read_u8()); + + if value_type == TRAILER_START_MARKER { + return None; + } + + let seqno = unwrap!(reader.read_u64_varint()); + + let key_len: usize = + fixed_key_len.map_or_else(|| unwrap!(reader.read_u16_varint()).into(), usize::from); + #[expect( + clippy::cast_possible_truncation, + reason = "blocks tend to be some megabytes in size at most, so position should fit into usize" + )] + let key_start = offset + reader.position() as usize; + #[expect( + clippy::cast_possible_wrap, + reason = "key_len is bounded by u16::MAX, no wrap expected" + )] + let key_len_i64 = key_len as i64; + unwrap!(reader.seek_relative(key_len_i64)); + + let key = data.get(key_start..(key_start + key_len)); + + key.map(|k| (k, seqno)) + } + + fn parse_full( + reader: &mut Cursor<&[u8]>, + offset: usize, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> Option { + let value_type = unwrap!(reader.read_u8()); + if value_type == TRAILER_START_MARKER { + return None; + } + + #[expect(clippy::expect_used, reason = "value_type is expected to be valid")] + let value_type = ValueType::try_from(value_type).expect("should be valid value type"); + + let seqno = unwrap!(reader.read_u64_varint()); + + let key_len: usize = + fixed_key_len.map_or_else(|| unwrap!(reader.read_u16_varint()).into(), usize::from); + #[expect( + clippy::cast_possible_truncation, + reason = "blocks tend to be some megabytes in size at most, so position should fit into usize" + )] + let key_start = offset + reader.position() as usize; + #[expect( + clippy::cast_possible_wrap, + reason = "key_len is bounded by u16::MAX, no wrap expected" + )] + let key_len_i64 = key_len as i64; + unwrap!(reader.seek_relative(key_len_i64)); + + let is_value = !value_type.is_tombstone(); + + let val_len: usize = if is_value { + fixed_value_len.map_or_else( + || unwrap!(reader.read_u32_varint()) as usize, + |len| len as usize, + ) + } else { + 0 + }; + #[expect( + clippy::cast_possible_truncation, + reason = "blocks tend to be some megabytes in size at most, so position should fit into usize" + )] + let val_offset = offset + reader.position() as usize; + #[expect( + clippy::cast_possible_wrap, + reason = "val_len is bounded by u32::MAX, no wrap expected" + )] + let val_len_i64 = val_len as i64; + unwrap!(reader.seek_relative(val_len_i64)); + + Some(if is_value { + DataBlockParsedItem { + value_type, + seqno, + prefix: None, + key: SliceIndexes(key_start, key_start + key_len), + value: Some(SliceIndexes(val_offset, val_offset + val_len)), + } + } else { + DataBlockParsedItem { + value_type, + seqno, + prefix: None, + key: SliceIndexes(key_start, key_start + key_len), + value: None, // TODO: enum value/tombstone, so value is not Option for values + } + }) + } + + fn parse_truncated( + reader: &mut Cursor<&[u8]>, + offset: usize, + base_key_offset: usize, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> Option { + let value_type = unwrap!(reader.read_u8()); + if value_type == TRAILER_START_MARKER { + return None; + } + let value_type = unwrap!(ValueType::try_from(value_type)); + + let seqno = unwrap!(reader.read_u64_varint()); + + let shared_prefix_len: usize = unwrap!(reader.read_u16_varint()).into(); + let rest_key_len: usize = fixed_key_len.map_or_else( + || unwrap!(reader.read_u16_varint()).into(), + |len| usize::from(len) - shared_prefix_len, + ); + + #[expect( + clippy::cast_possible_truncation, + reason = "truncation is not expected to happen" + )] + let key_offset = offset + reader.position() as usize; + + #[expect( + clippy::cast_possible_wrap, + reason = "rest_key_len is bounded by u16::MAX, no wrap expected" + )] + let rest_key_len_i64 = rest_key_len as i64; + unwrap!(reader.seek_relative(rest_key_len_i64)); + + let is_value = !value_type.is_tombstone(); + + let val_len: usize = if is_value { + fixed_value_len.map_or_else( + || unwrap!(reader.read_u32_varint()) as usize, + |len| len as usize, + ) + } else { + 0 + }; + #[expect( + clippy::cast_possible_truncation, + reason = "truncation is not expected to happen" + )] + let val_offset = offset + reader.position() as usize; + #[expect( + clippy::cast_possible_wrap, + reason = "val_len is bounded by u16::MAX, no wrap expected" + )] + let val_len_i64 = val_len as i64; + unwrap!(reader.seek_relative(val_len_i64)); + + Some(if is_value { + DataBlockParsedItem { + value_type, + seqno, + prefix: Some(SliceIndexes( + base_key_offset, + base_key_offset + shared_prefix_len, + )), + key: SliceIndexes(key_offset, key_offset + rest_key_len), + value: Some(SliceIndexes(val_offset, val_offset + val_len)), + } + } else { + DataBlockParsedItem { + value_type, + seqno, + prefix: Some(SliceIndexes( + base_key_offset, + base_key_offset + shared_prefix_len, + )), + key: SliceIndexes(key_offset, key_offset + rest_key_len), + value: None, + } + }) + } +} + +impl Encodable<()> for InternalValue { + fn encode_full_into( + &self, + writer: &mut W, + _state: &mut (), + fixed_key_len: Option, + fixed_value_len: Option, + ) -> crate::Result<()> { + // We encode restart markers as: + // [value type] [seqno] [user key len] [user key] [value len] [value] + // 1 2 3 4 5? 6? + + writer.write_u8(u8::from(self.key.value_type))?; // 1 + writer.write_u64_varint(self.key.seqno)?; // 2 + + if fixed_key_len.is_none() { + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 length max")] + writer.write_u16_varint(self.key.user_key.len() as u16)?; // 3 + } + writer.write_all(&self.key.user_key)?; // 4 + + // NOTE: Only write value len + value if we are actually a value + if !self.is_tombstone() { + if fixed_value_len.is_none() { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + writer.write_u32_varint(self.value.len() as u32)?; // 5 + } + writer.write_all(&self.value)?; // 6 + } + + Ok(()) + } + + fn encode_truncated_into( + &self, + writer: &mut W, + _state: &mut (), + shared_len: usize, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> crate::Result<()> { + // We encode truncated values as: + // [value type] [seqno] [shared prefix len] [rest key len] [rest key] [value len] [value] + // 1 2 3 4 5 6? 7? + + writer.write_u8(u8::from(self.key.value_type))?; // 1 + writer.write_u64_varint(self.key.seqno)?; // 2 + + // TODO: maybe we can skip this varint altogether if prefix truncation = false + + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 length max")] + writer.write_u16_varint(shared_len as u16)?; // 3 + + let rest_len = self.key().len() - shared_len; + + if fixed_key_len.is_none() { + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 length max")] + writer.write_u16_varint(rest_len as u16)?; // 4 + } + + #[expect( + clippy::expect_used, + reason = "the shared len should not be greater than key length" + )] + let truncated_user_key = self + .key + .user_key + .get(shared_len..) + .expect("should be in bounds"); + + writer.write_all(truncated_user_key)?; // 5 + + // NOTE: Only write value len + value if we are actually a value + if !self.is_tombstone() { + if fixed_value_len.is_none() { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + writer.write_u32_varint(self.value.len() as u32)?; // 6 + } + writer.write_all(&self.value)?; // 7 + } + + Ok(()) + } + + fn key(&self) -> &[u8] { + &self.key.user_key + } +} + +#[derive(Debug)] +pub struct DataBlockParsedItem { + pub value_type: ValueType, + pub seqno: SeqNo, + pub prefix: Option, + pub key: SliceIndexes, + pub value: Option, +} + +impl ParsedItem for DataBlockParsedItem { + fn compare_key(&self, needle: &[u8], bytes: &[u8]) -> std::cmp::Ordering { + if let Some(prefix) = &self.prefix { + let prefix = unsafe { bytes.get_unchecked(prefix.0..prefix.1) }; + let rest_key = unsafe { bytes.get_unchecked(self.key.0..self.key.1) }; + compare_prefixed_slice(prefix, rest_key, needle) + } else { + let key = unsafe { bytes.get_unchecked(self.key.0..self.key.1) }; + key.cmp(needle) + } + } + + fn key_offset(&self) -> usize { + self.key.0 + } + + fn materialize(&self, bytes: &Slice) -> InternalValue { + // NOTE: We consider the prefix and key slice indexes to be trustworthy + #[expect(clippy::indexing_slicing)] + let key = if let Some(prefix) = &self.prefix { + let prefix_key = &bytes[prefix.0..prefix.1]; + let rest_key = &bytes[self.key.0..self.key.1]; + Slice::fused(prefix_key, rest_key) + } else { + bytes.slice(self.key.0..self.key.1) + }; + + let key = InternalKey::new(key, self.seqno, self.value_type); + + let value = self + .value + .as_ref() + .map_or_else(Slice::empty, |v| bytes.slice(v.0..v.1)); + + InternalValue { key, value } + } +} + +impl DataBlockParsedItem { + fn materialize_value(&self, bytes: &Slice) -> PointReadValue { + let value = self + .value + .as_ref() + .map_or_else(Slice::empty, |value| bytes.slice(value.0..value.1)); + + PointReadValue { + value_type: self.value_type, + value, + } + } +} + +// TODO: allow disabling binary index (for meta block) +// -> saves space in metadata blocks +// -> point reads then need to use iter().find() to find stuff (which is fine) +// see https://github.com/fjall-rs/lsm-tree/issues/185 + +/// Block that contains key-value pairs (user data) +#[derive(Clone)] +pub struct DataBlock { + pub inner: Block, +} + +impl DataBlock { + /// Interprets a block as a data block. + /// + /// The caller needs to make sure the block is actually a data block + /// (e.g. by checking the block type, this is typically done in the `load_block` routine) + #[must_use] + pub fn new(inner: Block) -> Self { + Self { inner } + } + + /// Accesses the inner raw bytes + #[must_use] + pub fn as_slice(&self) -> &Slice { + &self.inner.data + } + + pub(crate) fn get_binary_index_reader(&self) -> BinaryIndexReader<'_> { + use std::mem::size_of; + + let trailer = Trailer::new(&self.inner); + + // NOTE: Skip restart interval (u8) + let offset = size_of::(); + + let mut reader = unwrap!(trailer.as_slice().get(offset..)); + + let binary_index_step_size = unwrap!(reader.read_u8()); + + debug_assert!( + binary_index_step_size == 2 || binary_index_step_size == 4, + "invalid binary index step size", + ); + + let binary_index_len = unwrap!(reader.read_u32::()); + let binary_index_offset = unwrap!(reader.read_u32::()); + + BinaryIndexReader::new( + &self.inner.data, + binary_index_offset, + binary_index_len, + binary_index_step_size, + ) + } + + #[must_use] + pub fn get_hash_index_reader(&self) -> Option> { + use std::mem::size_of; + + let trailer = Trailer::new(&self.inner); + + // NOTE: Skip restart interval (u8), binary index step size (u8) + // and binary stuff (2x u32) + let offset = size_of::() + size_of::() + size_of::() + size_of::(); + + let mut reader = unwrap!(trailer.as_slice().get(offset..)); + + let hash_index_len = unwrap!(reader.read_u32::()); + let hash_index_offset = unwrap!(reader.read_u32::()); + + if hash_index_len == 0 { + debug_assert_eq!( + 0, hash_index_offset, + "hash index offset should be 0 if its length is 0" + ); + None + } else { + Some(HashIndexReader::new( + &self.inner.data, + hash_index_offset, + hash_index_len, + )) + } + } + + /// Returns the number of hash buckets. + #[must_use] + pub fn hash_bucket_count(&self) -> Option { + self.get_hash_index_reader() + .map(|reader| reader.bucket_count()) + } + + // TODO: handle seqno more nicely (make Key generic, so we can do binary search over (key, seqno)) + #[must_use] + pub fn point_read(&self, needle: &[u8], seqno: SeqNo) -> Option { + self.point_read_item(needle, seqno) + .map(|item| item.materialize(&self.inner.data)) + } + + pub(crate) fn point_read_value(&self, needle: &[u8], seqno: SeqNo) -> Option { + self.point_read_item(needle, seqno) + .map(|item| item.materialize_value(&self.inner.data)) + } + + fn point_read_item(&self, needle: &[u8], seqno: SeqNo) -> Option { + let iter = if let Some(hash_index_reader) = self.get_hash_index_reader() { + match hash_index_reader.get(needle) { + MARKER_FREE => { + return None; + } + MARKER_CONFLICT => { + // NOTE: Fallback to binary search + let mut iter = self.iter(); + + if !iter.seek(needle) { + return None; + } + + iter + } + idx => { + let offset: usize = self.get_binary_index_reader().get(usize::from(idx)); + + let mut iter = self.iter(); + iter.seek_to_offset(offset); + + iter + } + } + } else { + let mut iter = self.iter(); + + // NOTE: Fallback to binary search + if !iter.seek(needle) { + return None; + } + + iter + }; + + // Linear scan + for item in iter { + match item.compare_key(needle, &self.inner.data) { + std::cmp::Ordering::Greater => { + // We are before our searched key/seqno + return None; + } + std::cmp::Ordering::Equal => { + // If key is same as needle, check sequence number + } + std::cmp::Ordering::Less => { + // We are past our searched key + continue; + } + } + + if item.seqno >= seqno { + continue; + } + + return Some(item); + } + + None + } + + #[must_use] + #[expect(clippy::iter_without_into_iter)] + pub fn iter(&self) -> Iter<'_> { + Iter::new( + &self.inner.data, + Decoder::::new(&self.inner), + ) + } + + /// Returns the binary index length (number of pointers). + /// + /// The number of pointers is equal to the number of restart intervals. + #[must_use] + pub fn binary_index_len(&self) -> u32 { + use std::mem::size_of; + + let trailer = Trailer::new(&self.inner); + + // NOTE: Skip restart interval (u8) and binary index step size (u8) + let offset = 2 * size_of::(); + let mut reader = unwrap!(trailer.as_slice().get(offset..)); + + unwrap!(reader.read_u32::()) + } + + /// Returns the number of items in the block. + #[must_use] + #[expect(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + Trailer::new(&self.inner).item_count() + } + + pub fn encode_into_vec( + items: &[InternalValue], + restart_interval: u8, + hash_index_ratio: f32, + ) -> crate::Result> { + let mut buf = vec![]; + + Self::encode_into(&mut buf, items, restart_interval, hash_index_ratio)?; + + Ok(buf) + } + + /// Builds an data block. + /// + /// # Panics + /// + /// Panics if the given item array if empty. + pub fn encode_into( + writer: &mut Vec, + items: &[InternalValue], + restart_interval: u8, + hash_index_ratio: f32, + ) -> crate::Result<()> { + #[expect(clippy::expect_used, reason = "the chunk should not be empty")] + let first_key = &items + .first() + .expect("chunk should not be empty") + .key + .user_key; + + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 length max")] + let fixed_key_len = items + .iter() + .map(|item| item.key.user_key.len()) + .all(|len| len == first_key.len()) + .then_some(first_key.len() as u16); + + let first_value_len = items + .iter() + .find(|item| !item.is_tombstone()) + .map(|item| item.value.len()); + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + let fixed_value_len = first_value_len.and_then(|first_len| { + items + .iter() + .filter(|item| !item.is_tombstone()) + .all(|item| item.value.len() == first_len) + .then_some(first_len as u32) + }); + + Self::encode_into_with_fixed_lengths( + writer, + items, + restart_interval, + hash_index_ratio, + fixed_key_len, + fixed_value_len, + ) + } + + pub(crate) fn encode_into_with_fixed_lengths( + writer: &mut Vec, + items: &[InternalValue], + restart_interval: u8, + hash_index_ratio: f32, + fixed_key_len: Option, + fixed_value_len: Option, + ) -> crate::Result<()> { + #[expect(clippy::expect_used, reason = "the chunk should not be empty")] + let first_key = &items + .first() + .expect("chunk should not be empty") + .key + .user_key; + + let mut serializer = Encoder::<'_, (), InternalValue>::new( + writer, + items.len(), + restart_interval, + hash_index_ratio, + first_key, + ) + .use_fixed_key_len(fixed_key_len) + .use_fixed_value_len(fixed_value_len); + + for item in items { + serializer.write(item)?; + } + + serializer.finish() + } +} + +#[cfg(test)] +#[expect(clippy::expect_used)] +mod tests { + use crate::{ + Checksum, InternalValue, SeqNo, Slice, + ValueType::{Tombstone, Value}, + table::{ + Block, DataBlock, + block::{BlockType, Header, ParsedItem}, + }, + }; + use test_log::test; + + #[test] + fn fixed_width_block_roundtrip_is_smaller() -> crate::Result<()> { + let items: Vec<_> = (0..256u64) + .map(|i| { + InternalValue::from_components(i.to_be_bytes(), (i as u32).to_be_bytes(), 0, Value) + }) + .collect(); + + let fixed = DataBlock::encode_into_vec(&items, 16, 0.0)?; + let mut legacy = Vec::new(); + DataBlock::encode_into_with_fixed_lengths(&mut legacy, &items, 16, 0.0, None, None)?; + + assert!( + fixed.len() + items.len() < legacy.len(), + "fixed-width encoding should omit at least one length per item", + ); + + for bytes in [fixed, legacy] { + let block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + let decoded: Vec<_> = block + .iter() + .map(|item| item.materialize(block.as_slice())) + .collect(); + assert_eq!(items, decoded); + } + + Ok(()) + } + + #[test] + fn data_block_ping_pong_fuzz_1() -> crate::Result<()> { + let items = [ + InternalValue::from_components( + Slice::from([111]), + Slice::from([119]), + 8_602_264_972_526_186_597, + Value, + ), + InternalValue::from_components( + Slice::from([121, 120, 99]), + Slice::from([101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101]), + 11_426_548_769_907, + Value, + ), + ]; + + let ping_pong_code = [1, 0]; + + let bytes: Vec = DataBlock::encode_into_vec(&items, 1, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let expected_ping_ponged_items = { + let mut iter = items.iter(); + let mut v = vec![]; + + for &x in &ping_pong_code { + if x == 0 { + v.push(iter.next().cloned().expect("should have item")); + } else { + v.push(iter.next_back().cloned().expect("should have item")); + } + } + + v + }; + + let real_ping_ponged_items = { + let mut iter = data_block + .iter() + .map(|x| x.materialize(data_block.as_slice())); + + let mut v = vec![]; + + for &x in &ping_pong_code { + if x == 0 { + v.push(iter.next().expect("should have item")); + } else { + v.push(iter.next_back().expect("should have item")); + } + } + + v + }; + + assert_eq!(expected_ping_ponged_items, real_ping_ponged_items); + + Ok(()) + } + + #[test] + fn data_block_point_read_simple() -> crate::Result<()> { + let items = [ + InternalValue::from_components("b", "b", 0, Value), + InternalValue::from_components("c", "c", 0, Value), + InternalValue::from_components("d", "d", 1, Tombstone), + InternalValue::from_components("e", "e", 0, Value), + InternalValue::from_components("f", "f", 0, Value), + ]; + + for restart_interval in 1..=16 { + let bytes: Vec = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert!( + data_block.point_read(b"a", SeqNo::MAX).is_none(), + "should return None because a does not exist", + ); + + assert!( + data_block.point_read(b"b", SeqNo::MAX).is_some(), + "should return Some because b exists", + ); + + assert!( + data_block.point_read(b"z", SeqNo::MAX).is_none(), + "should return Some because z does not exist", + ); + } + + Ok(()) + } + + #[test] + fn data_block_point_read_one() -> crate::Result<()> { + let items = [InternalValue::from_components( + "pla:earth:fact", + "eaaaaaaaaarth", + 0, + crate::ValueType::Value, + )]; + + let bytes = DataBlock::encode_into_vec(&items, 16, 0.0)?; + let serialized_len = bytes.len(); + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert_eq!(data_block.inner.size(), serialized_len); + assert_eq!(1, data_block.binary_index_len()); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, SeqNo::MAX), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + fn data_block_vhandle() -> crate::Result<()> { + let items = [InternalValue::from_components( + "abc", + "world", + 1, + crate::ValueType::Indirection, + )]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + let serialized_len = bytes.len(); + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert_eq!(data_block.inner.size(), serialized_len); + + assert_eq!(Some(items[0].clone()), data_block.point_read(b"abc", 777)); + assert!(data_block.point_read(b"abc", 1).is_none()); + } + + Ok(()) + } + + #[test] + fn data_block_mvcc_read_first() -> crate::Result<()> { + let items = [InternalValue::from_components( + "hello", + "world", + 0, + crate::ValueType::Value, + )]; + + for restart_interval in 1..=16 { + let bytes = DataBlock::encode_into_vec(&items, restart_interval, 0.0)?; + let serialized_len = bytes.len(); + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert_eq!(data_block.inner.size(), serialized_len); + + assert_eq!(Some(items[0].clone()), data_block.point_read(b"hello", 777)); + } + + Ok(()) + } + + #[test] + fn data_block_point_read_fuzz_1() -> crate::Result<()> { + let items = [ + InternalValue::from_components([0], b"", 23_523_531_241_241_242, Value), + InternalValue::from_components([0], b"", 0, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 16, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!( + data_block + .hash_bucket_count() + .expect("should have built hash index") + > 0, + ); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, needle.key.seqno + 1), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + fn data_block_point_read_fuzz_2() -> crate::Result<()> { + let items = [ + InternalValue::from_components([0], [], 5, Value), + InternalValue::from_components([0], [], 4, Tombstone), + InternalValue::from_components([0], [], 3, Value), + InternalValue::from_components([0], [], 0, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 2, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!(data_block.hash_bucket_count().is_none()); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, needle.key.seqno + 1), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + fn data_block_point_read_dense() -> crate::Result<()> { + let items = [ + InternalValue::from_components(b"a", b"a", 3, Value), + InternalValue::from_components(b"b", b"b", 2, Value), + InternalValue::from_components(b"c", b"c", 1, Value), + InternalValue::from_components(b"d", b"d", 65, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 1, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert_eq!(4, data_block.binary_index_len()); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, SeqNo::MAX), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + fn data_block_point_read_dense_mvcc_with_hash() -> crate::Result<()> { + let items = [ + InternalValue::from_components(b"a", b"a", 3, Value), + InternalValue::from_components(b"a", b"a", 2, Value), + InternalValue::from_components(b"a", b"a", 1, Value), + InternalValue::from_components(b"b", b"b", 65, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 1, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!( + data_block + .hash_bucket_count() + .expect("should have built hash index") + > 0, + ); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, needle.key.seqno + 1), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn data_block_point_read_mvcc_latest_fuzz_1() -> crate::Result<()> { + let items = [ + InternalValue::from_components(Slice::from([0]), Slice::from([]), 0, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 0, Value), + InternalValue::from_components( + Slice::from([255, 255, 0]), + Slice::from([]), + 127_886_946_205_696, + Tombstone, + ), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 2, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!(data_block.get_hash_index_reader().is_none()); + + assert_eq!( + Some(items.get(1).cloned().unwrap()), + data_block.point_read(&[233, 233], SeqNo::MAX) + ); + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn data_block_point_read_mvcc_latest_fuzz_2() -> crate::Result<()> { + let items = [ + InternalValue::from_components(Slice::from([0]), Slice::from([]), 0, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 8, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 7, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 6, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 5, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 4, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 3, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 2, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 1, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 0, Value), + InternalValue::from_components( + Slice::from([255, 255, 0]), + Slice::from([]), + 127_886_946_205_696, + Tombstone, + ), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 2, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + + assert_eq!( + Some(items.get(1).cloned().unwrap()), + data_block.point_read(&[233, 233], SeqNo::MAX) + ); + assert_eq!( + Some(items.last().cloned().unwrap()), + data_block.point_read(&[255, 255, 0], SeqNo::MAX) + ); + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn data_block_point_read_mvcc_latest_fuzz_3() -> crate::Result<()> { + let items = [ + InternalValue::from_components(Slice::from([0]), Slice::from([]), 0, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 8, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 7, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 6, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 5, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 4, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 3, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 2, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 1, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 0, Value), + InternalValue::from_components( + Slice::from([255, 255, 0]), + Slice::from([]), + 127_886_946_205_696, + Tombstone, + ), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 2, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + + assert_eq!( + Some(items.get(1).cloned().unwrap()), + data_block.point_read(&[233, 233], SeqNo::MAX) + ); + assert_eq!( + Some(items.last().cloned().unwrap()), + data_block.point_read(&[255, 255, 0], SeqNo::MAX) + ); + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + #[expect(clippy::unwrap_used)] + fn data_block_point_read_mvcc_latest_fuzz_3_dense() -> crate::Result<()> { + let items = [ + InternalValue::from_components(Slice::from([0]), Slice::from([]), 0, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 8, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 7, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 6, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 5, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 4, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 3, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 2, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 1, Value), + InternalValue::from_components(Slice::from([233, 233]), Slice::from([]), 0, Value), + InternalValue::from_components( + Slice::from([255, 255, 0]), + Slice::from([]), + 127_886_946_205_696, + Tombstone, + ), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 1, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + + assert_eq!( + Some(items.get(1).cloned().unwrap()), + data_block.point_read(&[233, 233], SeqNo::MAX) + ); + assert_eq!( + Some(items.last().cloned().unwrap()), + data_block.point_read(&[255, 255, 0], SeqNo::MAX) + ); + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + fn data_block_point_read_dense_mvcc_no_hash() -> crate::Result<()> { + let items = [ + InternalValue::from_components(b"a", b"a", 3, Value), + InternalValue::from_components(b"a", b"a", 2, Value), + InternalValue::from_components(b"a", b"a", 1, Value), + InternalValue::from_components(b"b", b"b", 65, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 1, 0.0)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!(data_block.hash_bucket_count().is_none()); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, needle.key.seqno + 1), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } + + #[test] + fn data_block_point_read_shadowing() -> crate::Result<()> { + let items = [ + InternalValue::from_components("pla:saturn:fact", "Saturn is pretty big", 0, Value), + InternalValue::from_components("pla:saturn:name", "Saturn", 0, Value), + InternalValue::from_components("pla:venus:fact", "", 1, Tombstone), + InternalValue::from_components("pla:venus:fact", "Venus exists", 0, Value), + InternalValue::from_components("pla:venus:name", "Venus", 0, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 16, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!( + data_block + .hash_bucket_count() + .expect("should have built hash index") + > 0, + ); + + assert!( + data_block + .point_read(b"pla:venus:fact", SeqNo::MAX) + .expect("should exist") + .is_tombstone() + ); + + Ok(()) + } + + #[test] + fn data_block_point_read_dense_2() -> crate::Result<()> { + let items = [ + InternalValue::from_components("pla:earth:fact", "eaaaaaaaaarth", 0, Value), + InternalValue::from_components("pla:jupiter:fact", "Jupiter is big", 0, Value), + InternalValue::from_components("pla:jupiter:mass", "Massive", 0, Value), + InternalValue::from_components("pla:jupiter:name", "Jupiter", 0, Value), + InternalValue::from_components("pla:jupiter:radius", "Big", 0, Value), + InternalValue::from_components("pla:saturn:fact", "Saturn is pretty big", 0, Value), + InternalValue::from_components("pla:saturn:name", "Saturn", 0, Value), + InternalValue::from_components("pla:venus:fact", "", 1, Tombstone), + InternalValue::from_components("pla:venus:fact", "Venus exists", 0, Value), + InternalValue::from_components("pla:venus:name", "Venus", 0, Value), + ]; + + let bytes = DataBlock::encode_into_vec(&items, 1, 1.33)?; + + let data_block = DataBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Data, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(data_block.len(), items.len()); + assert!( + data_block + .hash_bucket_count() + .expect("should have built hash index") + > 0, + ); + + for needle in items { + assert_eq!( + Some(needle.clone()), + data_block.point_read(&needle.key.user_key, needle.key.seqno + 1), + ); + } + + assert_eq!(None, data_block.point_read(b"yyy", SeqNo::MAX)); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/filter/bit_array/builder.rs b/crates/lsm-tree/src/table/filter/bit_array/builder.rs new file mode 100644 index 000000000..9df79c28b --- /dev/null +++ b/crates/lsm-tree/src/table/filter/bit_array/builder.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +const BIT_MASK: u8 = 0b1000_0000_u8; + +/// Sets a bit in the byte to `true` +#[must_use] +pub fn enable_bit(byte: u8, idx: usize) -> u8 { + let bit_mask = BIT_MASK >> idx; + byte | bit_mask +} + +/// Fixed-size bit array +#[derive(Debug)] +pub struct Builder(Box<[u8]>); + +impl Builder { + #[must_use] + pub fn with_capacity(bytes: usize) -> Self { + let vec = vec![0; bytes]; + Self(vec.into_boxed_slice()) + } + + #[must_use] + pub fn from_bytes(bytes: Box<[u8]>) -> Self { + Self(bytes) + } + + #[must_use] + pub fn bytes(&self) -> &[u8] { + &self.0 + } + + /// Sets the i-th bit + pub fn enable_bit(&mut self, idx: usize) { + let byte_idx = idx / 8; + + #[expect(clippy::expect_used, reason = "we trust the caller")] + let byte = self.0.get_mut(byte_idx).expect("should be in bounds"); + + let bit_idx = idx % 8; + *byte = enable_bit(*byte, bit_idx); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn bit_set_true() { + assert_eq!(0b0000_0010, enable_bit(0, 6)); + assert_eq!(0b1000_0000, enable_bit(0, 0)); + assert_eq!(0b0100_0000, enable_bit(0, 1)); + assert_eq!(0b0100_0110, enable_bit(0b0000_0110, 1)); + } + + #[test] + fn bit_array_builder_basic() { + let mut builder = Builder::with_capacity(1); + assert_eq!(&[0], builder.bytes()); + + builder.enable_bit(0); + assert_eq!(&[0b1000_0000], builder.bytes()); + + builder.enable_bit(7); + assert_eq!(&[0b1000_0001], builder.bytes()); + } +} diff --git a/crates/lsm-tree/src/table/filter/bit_array/mod.rs b/crates/lsm-tree/src/table/filter/bit_array/mod.rs new file mode 100644 index 000000000..061a2cd34 --- /dev/null +++ b/crates/lsm-tree/src/table/filter/bit_array/mod.rs @@ -0,0 +1,9 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod builder; +mod reader; + +pub use builder::Builder; +pub use reader::BitArrayReader; diff --git a/crates/lsm-tree/src/table/filter/bit_array/reader.rs b/crates/lsm-tree/src/table/filter/bit_array/reader.rs new file mode 100644 index 000000000..bf2ea88d9 --- /dev/null +++ b/crates/lsm-tree/src/table/filter/bit_array/reader.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +const BIT_MASK: u8 = 0b1000_0000_u8; + +/// Gets a bit from the byte +fn get_bit(byte: u8, idx: usize) -> bool { + let bit_mask = BIT_MASK >> idx; + + let masked = byte & bit_mask; + masked > 0 +} + +/// Fixed-size bit array reader +#[derive(Debug)] +pub struct BitArrayReader<'a>(&'a [u8]); + +impl<'a> BitArrayReader<'a> { + #[must_use] + pub fn new(bytes: &'a [u8]) -> Self { + Self(bytes) + } + + #[must_use] + pub fn bytes(&self) -> &[u8] { + self.0 + } + + /// Gets the i-th bit. + #[must_use] + pub fn get(&self, idx: usize) -> bool { + let byte_idx = idx / 8; + + #[expect(clippy::expect_used, reason = "we trust the caller")] + let byte = self.0.get(byte_idx).expect("should be in bounds"); + + let bit_idx = idx % 8; + get_bit(*byte, bit_idx) + } +} diff --git a/crates/lsm-tree/src/table/filter/block.rs b/crates/lsm-tree/src/table/filter/block.rs new file mode 100644 index 000000000..2dc43b512 --- /dev/null +++ b/crates/lsm-tree/src/table/filter/block.rs @@ -0,0 +1,25 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::table::{Block, filter::standard_bloom::StandardBloomFilterReader}; + +#[derive(Clone)] +pub struct FilterBlock(Block); + +impl FilterBlock { + #[must_use] + pub fn new(block: Block) -> Self { + Self(block) + } + + pub fn maybe_contains_hash(&self, hash: u64) -> crate::Result { + Ok(StandardBloomFilterReader::new(&self.0.data)?.contains_hash(hash)) + } + + /// Returns the block size in bytes. + #[must_use] + pub fn size(&self) -> usize { + self.0.size() + } +} diff --git a/crates/lsm-tree/src/table/filter/blocked_bloom/builder._rs b/crates/lsm-tree/src/table/filter/blocked_bloom/builder._rs new file mode 100644 index 000000000..4aac77c8b --- /dev/null +++ b/crates/lsm-tree/src/table/filter/blocked_bloom/builder._rs @@ -0,0 +1,169 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::super::bit_array::Builder as BitArrayBuilder; +use super::super::standard_bloom::builder::secondary_hash; +use crate::{ + file::MAGIC_BYTES, + table::filter::{blocked_bloom::CACHE_LINE_BYTES, FilterType}, +}; +use byteorder::{LittleEndian, WriteBytesExt}; +use std::io::Write; + +#[derive(Debug)] +pub struct Builder { + /// Raw bytes exposed as bit array + inner: BitArrayBuilder, + + /// Number of hash functions + pub(crate) k: usize, + + /// Number of blocks in the blocked bloom filter + pub(crate) num_blocks: usize, +} + +impl Builder { + #[expect( + clippy::expect_used, + reason = "we write into a Vec, so no I/O error can happen" + )] + #[must_use] + pub fn build(&self) -> Vec { + let mut v = vec![]; + + // Write header + v.write_all(&MAGIC_BYTES).expect("should not fail"); + + // NOTE: Filter type + v.write_u8(FilterType::BlockedBloom.into()) + .expect("should not fail"); + + // NOTE: Hash type (unused) + v.write_u8(0).expect("should not fail"); + + v.write_u64::(self.num_blocks as u64) + .expect("should not fail"); + v.write_u64::(self.k as u64) + .expect("should not fail"); + v.write_all(self.inner.bytes()).expect("should not fail"); + + v + } + + /// Constructs a bloom filter that can hold `n` items + /// while maintaining a certain false positive rate `fpr`. + #[must_use] + pub fn with_fp_rate(n: usize, fpr: f32) -> Self { + use std::f32::consts::LN_2; + + assert!(n > 0); + + // NOTE: Some sensible minimum + let fpr = fpr.max(0.000_000_1); + + // NOTE: We add ~5-25% more bits to account for blocked bloom filters being a bit less accurate + // See https://dl.acm.org/doi/10.1145/1498698.1594230 + let bonus = match fpr { + _ if fpr <= 0.001 => 1.25, + _ if fpr <= 0.01 => 1.2, + _ if fpr <= 0.1 => 1.1, + _ => 1.05, + }; + + let m = ((Self::calculate_m(n, fpr)) as f32 * bonus) as usize; + let bpk = m / n; + let k = (((bpk as f32) * LN_2) as usize).max(1); + + let num_blocks = m.div_ceil(CACHE_LINE_BYTES * 8); + + Self { + inner: BitArrayBuilder::with_capacity(num_blocks * CACHE_LINE_BYTES), + k, + num_blocks, + } + } + + /// Constructs a bloom filter that can hold `n` items + /// with `bpk` bits per key. + /// + /// 10 bits per key is a sensible default. + #[must_use] + pub fn with_bpk(n: usize, bpk: u8) -> Self { + use std::f32::consts::LN_2; + + assert!(bpk > 0); + assert!(n > 0); + + let bpk = bpk as usize; + + let m = n * bpk; + let k = (((bpk as f32) * LN_2) as usize).max(1); + + let num_blocks = m.div_ceil(CACHE_LINE_BYTES * 8); + + Self { + inner: BitArrayBuilder::with_capacity(num_blocks * CACHE_LINE_BYTES), + k, + num_blocks, + } + } + + fn calculate_m(n: usize, fp_rate: f32) -> usize { + use std::f32::consts::LN_2; + + let n = n as f32; + let ln2_squared = LN_2.powi(2); + + let numerator = n * fp_rate.ln(); + let m = -(numerator / ln2_squared); + + // Round up to next byte + ((m / 8.0).ceil() * 8.0) as usize + } + + /// Adds the key to the filter. + pub fn set_with_hash(&mut self, mut h1: u64) { + let mut h2 = secondary_hash(h1); + + let block_idx = h1 % (self.num_blocks as u64); + + for i in 1..(self.k as u64) { + let idx = h1 % (CACHE_LINE_BYTES as u64 * 8); + + #[expect( + clippy::cast_possible_truncation, + reason = "filters tend to be pretty small, definitely less than 4 GiB, even for large tables" + )] + self.inner + .enable_bit(Self::get_bit_idx(block_idx as usize, idx as usize)); + + h1 = h1.wrapping_add(h2); + h2 = h2.wrapping_mul(i); + } + } + + #[must_use] + pub fn get_bit_idx(block_idx: usize, idx_in_block: usize) -> usize { + block_idx * CACHE_LINE_BYTES * 8 + idx_in_block + } + + /// Gets the hash of a key. + #[must_use] + pub fn get_hash(key: &[u8]) -> u64 { + super::super::standard_bloom::Builder::get_hash(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn bloom_calculate_m() { + assert_eq!(9_592, Builder::calculate_m(1_000, 0.01)); + assert_eq!(4_800, Builder::calculate_m(1_000, 0.1)); + assert_eq!(4_792_536, Builder::calculate_m(1_000_000, 0.1)); + } +} diff --git a/crates/lsm-tree/src/table/filter/blocked_bloom/mod._rs b/crates/lsm-tree/src/table/filter/blocked_bloom/mod._rs new file mode 100644 index 000000000..da0fb689a --- /dev/null +++ b/crates/lsm-tree/src/table/filter/blocked_bloom/mod._rs @@ -0,0 +1,193 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod builder; + +pub use builder::Builder; + +use super::bit_array::BitArrayReader; +use crate::{ + file::MAGIC_BYTES, + table::filter::{standard_bloom::builder::secondary_hash, FilterType}, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::io::{Cursor, Read}; + +const CACHE_LINE_BYTES: usize = 64; + +/// A blocked bloom filter +/// +/// Allows buffering the key hashes before actual filter construction +/// which is needed to properly calculate the filter size, as the number of items +/// are unknown during table construction. +/// +/// The filter uses double hashing instead of `k` hash functions, see: +/// +#[derive(Debug)] +pub struct BlockedBloomFilterReader<'a> { + /// Raw bytes exposed as bit array + inner: BitArrayReader<'a>, + + /// Number of hash functions + pub(crate) k: usize, + + /// Number of blocks in the blocked bloom filter + pub(crate) num_blocks: usize, +} + +impl<'a> BlockedBloomFilterReader<'a> { + pub fn new(slice: &'a [u8]) -> crate::Result { + let mut reader = Cursor::new(slice); + + // Check header + let mut magic = [0u8; MAGIC_BYTES.len()]; + reader.read_exact(&mut magic)?; + + if magic != MAGIC_BYTES { + return Err(crate::Error::InvalidHeader("BloomFilter")); + } + + // NOTE: Filter type + let filter_type = reader.read_u8()?; + let filter_type = FilterType::try_from(filter_type)?; + assert_eq!( + FilterType::BlockedBloom, + filter_type, + "Invalid filter type, got={filter_type:?}, expected={:?}", + FilterType::BlockedBloom, + ); + + // NOTE: Hash type (unused) + let hash_type = reader.read_u8()?; + assert_eq!(0, hash_type, "Invalid bloom hash type"); + + let num_blocks = reader.read_u64::()? as usize; + let k = reader.read_u64::()? as usize; + + let offset = reader.position() as usize; + + Ok(Self { + k, + num_blocks, + inner: BitArrayReader::new(slice.get(offset..).expect("should be in bounds")), + }) + } + + fn bytes(&self) -> &[u8] { + self.inner.bytes() + } + + /// Size of bloom filter in bytes + #[must_use] + fn len(&self) -> usize { + self.inner.bytes().len() + } + + /// Returns `true` if the hash may be contained. + /// + /// Will never have a false negative. + #[must_use] + pub fn contains_hash(&self, mut h1: u64) -> bool { + let mut h2 = secondary_hash(h1); + + let block_idx = h1 % (self.num_blocks as u64); + + for i in 1..(self.k as u64) { + let bit_idx = h1 % (CACHE_LINE_BYTES as u64 * 8); + + if !self.has_bit(block_idx as usize, bit_idx as usize) { + return false; + } + + h1 = h1.wrapping_add(h2); + h2 = h2.wrapping_mul(i); + } + + true + } + + /// Returns `true` if the bit at `idx` is `1`. + fn has_bit(&self, block_idx: usize, idx_in_block: usize) -> bool { + self.inner + .get(Builder::get_bit_idx(block_idx, idx_in_block)) + } + + /// Gets the hash of a key. + #[must_use] + pub fn get_hash(key: &[u8]) -> u64 { + Builder::get_hash(key) + } + + /// Returns `true` if the item may be contained. + /// + /// Will never have a false negative. + #[must_use] + pub fn contains(&self, key: &[u8]) -> bool { + self.contains_hash(Self::get_hash(key)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn filter_bloom_blocked_serde_round_trip() -> crate::Result<()> { + let mut filter = Builder::with_fp_rate(10, 0.0001); + + let keys = &[ + b"item0", b"item1", b"item2", b"item3", b"item4", b"item5", b"item6", b"item7", + b"item8", b"item9", + ]; + + for key in keys { + filter.set_with_hash(BlockedBloomFilterReader::get_hash(*key)); + } + + let filter_bytes = filter.build(); + let filter_copy = BlockedBloomFilterReader::new(&filter_bytes)?; + + assert_eq!(filter.k, filter_copy.k); + assert_eq!(filter.num_blocks, filter_copy.num_blocks); + assert!(!filter_copy.contains(b"asdasads")); + assert!(!filter_copy.contains(b"item10")); + assert!(!filter_copy.contains(b"cxycxycxy")); + + Ok(()) + } + + #[test] + fn filter_bloom_blocked_basic() -> crate::Result<()> { + let mut filter = Builder::with_fp_rate(10, 0.0001); + + let keys = [ + b"item0" as &[u8], + b"item1", + b"item2", + b"item3", + b"item4", + b"item5", + b"item6", + b"item7", + b"item8", + b"item9", + ]; + + for key in &keys { + filter.set_with_hash(Builder::get_hash(key)); + } + + let filter_bytes = filter.build(); + let filter = BlockedBloomFilterReader::new(&filter_bytes)?; + + for key in &keys { + assert!(filter.contains(key)); + } + + assert!(!filter.contains(b"asdasdasdasdasdasdasd")); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/filter/mod.rs b/crates/lsm-tree/src/table/filter/mod.rs new file mode 100644 index 000000000..012d9ce3f --- /dev/null +++ b/crates/lsm-tree/src/table/filter/mod.rs @@ -0,0 +1,115 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod bit_array; +pub mod block; +// pub mod blocked_bloom; +pub mod standard_bloom; + +use standard_bloom::Builder as StandardBloomFilterBuilder; + +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum BloomConstructionPolicy { + BitsPerKey(f32), + FalsePositiveRate(f32), +} + +impl Default for BloomConstructionPolicy { + fn default() -> Self { + Self::BitsPerKey(10.0) + } +} + +impl BloomConstructionPolicy { + #[must_use] + pub fn init(&self, n: usize) -> StandardBloomFilterBuilder { + use standard_bloom::Builder; + + match self { + Self::BitsPerKey(bpk) => Builder::with_bpk(n, *bpk), + Self::FalsePositiveRate(fpr) => Builder::with_fp_rate(n, *fpr), + } + } + + #[must_use] + pub fn is_active(&self) -> bool { + match self { + Self::BitsPerKey(bpk) => *bpk > 0.0, + Self::FalsePositiveRate(fpr) => *fpr > 0.0, + } + } + + /// Returns the estimated filter size in bytes. + #[must_use] + pub fn estimated_filter_size(&self, n: usize) -> usize { + if n == 0 { + return 0; + } + + #[expect( + clippy::cast_precision_loss, + reason = "truncation is fine because this is an estimation" + )] + match self { + Self::BitsPerKey(bpk) => (*bpk * (n as f32)) as usize / 8, + Self::FalsePositiveRate(fpr) => { + let m = StandardBloomFilterBuilder::calculate_m(n, *fpr); + let bpk = (m / n) as f32; + (bpk * (n as f32)) as usize / 8 + } + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +enum FilterType { + StandardBloom, + BlockedBloom, +} + +impl TryFrom for FilterType { + type Error = crate::Error; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::StandardBloom), + 1 => Ok(Self::BlockedBloom), + _ => Err(crate::Error::InvalidTag(("FilterType", value))), + } + } +} + +impl From for u8 { + fn from(value: FilterType) -> Self { + match value { + FilterType::StandardBloom => 0, + FilterType::BlockedBloom => 1, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn bloom_estimated_size_bpk() { + let policy = BloomConstructionPolicy::BitsPerKey(10.0); + let n = 1_000_000; + let estimated_size = policy.estimated_filter_size(n); + // For 1 million keys and 10 bits per key, the size should be around 1.25 MB + assert_eq!(estimated_size, 1_250_000); + } + + #[test] + fn bloom_estimated_size_fpr() { + let policy = BloomConstructionPolicy::FalsePositiveRate(0.01); + let n = 1_000_000; + let estimated_size = policy.estimated_filter_size(n); + // For 1 million keys and 1% false positive rate, the size should be around 1.2 MB + assert!((estimated_size as f32) < 1_300_000.0); + assert!((estimated_size as f32) > 1_100_000.0); + } +} diff --git a/crates/lsm-tree/src/table/filter/standard_bloom/builder.rs b/crates/lsm-tree/src/table/filter/standard_bloom/builder.rs new file mode 100644 index 000000000..8295c4f15 --- /dev/null +++ b/crates/lsm-tree/src/table/filter/standard_bloom/builder.rs @@ -0,0 +1,188 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::super::bit_array::Builder as BitArrayBuilder; +use crate::{file::MAGIC_BYTES, table::filter::FilterType}; +use byteorder::{LittleEndian, WriteBytesExt}; +use std::io::Write; + +pub fn secondary_hash(h1: u64) -> u64 { + // Taken from https://github.com/tomtomwombat/fastbloom + h1.wrapping_shr(32).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95) +} + +#[derive(Debug)] +pub struct Builder { + /// Raw bytes exposed as bit array + inner: BitArrayBuilder, + + /// Bit count + pub(super) m: usize, + + /// Number of hash functions + pub(super) k: usize, +} + +impl Builder { + #[expect( + clippy::expect_used, + reason = "we write into a Vec, so no I/O error can happen" + )] + #[must_use] + pub fn build(&self) -> Vec { + let mut v = vec![]; + + // Write header + v.write_all(&MAGIC_BYTES).expect("should not fail"); + + // NOTE: Filter type + v.write_u8(FilterType::StandardBloom.into()) + .expect("should not fail"); + + // NOTE: Hash type (unused) + v.write_u8(0).expect("should not fail"); + + v.write_u64::(self.m as u64) + .expect("should not fail"); + v.write_u64::(self.k as u64) + .expect("should not fail"); + v.write_all(self.inner.bytes()).expect("should not fail"); + + v + } + + /// Constructs a bloom filter that can hold `n` items + /// while maintaining a certain false positive rate `fpr`. + #[must_use] + pub fn with_fp_rate(n: usize, fpr: f32) -> Self { + use std::f32::consts::LN_2; + + assert!(n > 0); + + // NOTE: Some sensible minimum + let fpr = fpr.max(0.000_000_1); + + let m = Self::calculate_m(n, fpr); + + #[expect( + clippy::cast_precision_loss, + reason = "bpk tends to be in the range of 0-50, so easily fits into u32" + )] + let bpk = (m / n) as f32; + + #[expect( + clippy::cast_sign_loss, + clippy::cast_possible_truncation, + reason = "bpk easily fits into u32 and LN_2 < 1.0, so should still fit into a usize as well" + )] + let k = ((bpk * LN_2) as usize).max(1); + + Self { + inner: BitArrayBuilder::with_capacity(m / 8), + m, + k, + } + } + + /// Constructs a bloom filter that can hold `n` items + /// with `bpk` bits per key. + /// + /// 10 bits per key is a sensible default. + #[must_use] + pub fn with_bpk(n: usize, bpk: f32) -> Self { + use std::f32::consts::LN_2; + + assert!(bpk > 0.0); + assert!(n > 0); + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "bpk tends to be in the range of 0-50, so easily fits into usize" + )] + let m = n * (bpk as usize); + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "bpk easily fits into usize and LN_2 < 1.0, so should still fit into a usize as well" + )] + let k = ((bpk * LN_2) as usize).max(1); + + // NOTE: Round up so we don't get too little bits + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss, + reason = "m already fits, and because we divide, it should definitely fit into usize" + )] + let bytes = (m as f32 / 8.0).ceil() as usize; + + Self { + inner: BitArrayBuilder::with_capacity(bytes), + m: bytes * 8, + k, + } + } + + pub(crate) fn calculate_m(n: usize, fp_rate: f32) -> usize { + use std::f32::consts::LN_2; + + #[expect( + clippy::cast_precision_loss, + reason = "n tends to be in the single millions at most, so f32 should be precise enough" + )] + let n = n as f32; + let ln2_squared = LN_2.powi(2); + + let numerator = n * fp_rate.ln(); + let m = -(numerator / ln2_squared); + + // NOTE: Round up to next byte + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "m already fits, and because we divide, it should definitely fit into usize" + )] + let result = ((m / 8.0).ceil() * 8.0) as usize; + result + } + + /// Adds the key to the filter. + pub fn set_with_hash(&mut self, mut h1: u64) { + let mut h2 = secondary_hash(h1); + + for i in 1..=(self.k as u64) { + let idx = h1 % (self.m as u64); + + #[expect( + clippy::cast_possible_truncation, + reason = "filters tend to be pretty small, definitely less than 4 GiB, even for large tables" + )] + self.inner.enable_bit(idx as usize); + + h1 = h1.wrapping_add(h2); + h2 = h2.wrapping_mul(i); + } + } + + /// Gets the hash of a key. + #[must_use] + pub fn get_hash(key: &[u8]) -> u64 { + crate::hash::hash64(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn bloom_calculate_m() { + assert_eq!(9_592, Builder::calculate_m(1_000, 0.01)); + assert_eq!(4_800, Builder::calculate_m(1_000, 0.1)); + assert_eq!(4_792_536, Builder::calculate_m(1_000_000, 0.1)); + } +} diff --git a/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs b/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs new file mode 100644 index 000000000..e47d000ad --- /dev/null +++ b/crates/lsm-tree/src/table/filter/standard_bloom/mod.rs @@ -0,0 +1,305 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub(crate) mod builder; + +pub use builder::Builder; + +use super::bit_array::BitArrayReader; +use crate::{ + file::MAGIC_BYTES, + table::filter::{FilterType, standard_bloom::builder::secondary_hash}, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::io::{Cursor, Read}; + +/// A standard bloom filter +/// +/// Allows buffering the key hashes before actual filter construction +/// which is needed to properly calculate the filter size, as the number of items +/// are unknown during table construction. +/// +/// The filter uses double hashing instead of `k` hash functions, see: +/// +pub struct StandardBloomFilterReader<'a> { + /// Raw bytes exposed as bit array + inner: BitArrayReader<'a>, + + /// Bit count + m: usize, + + /// Number of hash functions + k: usize, +} + +impl<'a> StandardBloomFilterReader<'a> { + pub fn new(slice: &'a [u8]) -> crate::Result { + let mut reader = Cursor::new(slice); + + // Check header + let mut magic = [0u8; MAGIC_BYTES.len()]; + reader.read_exact(&mut magic)?; + + if magic != MAGIC_BYTES { + return Err(crate::Error::InvalidHeader("BloomFilter")); + } + + // NOTE: Filter type + let filter_type = reader.read_u8()?; + let filter_type = FilterType::try_from(filter_type)?; + assert_eq!( + FilterType::StandardBloom, + filter_type, + "Invalid filter type, got={filter_type:?}, expected={:?}", + FilterType::StandardBloom + ); + + // NOTE: Hash type (unused) + let hash_type = reader.read_u8()?; + assert_eq!(0, hash_type, "Invalid bloom hash type"); + + #[expect( + clippy::cast_possible_truncation, + reason = "filters in a single table tend to be a couple of megabits of data at most, so easily fits into usize" + )] + let m = reader.read_u64::()? as usize; + + #[expect( + clippy::cast_possible_truncation, + reason = "k easily fits into any integer" + )] + let k = reader.read_u64::()? as usize; + + #[expect( + clippy::cast_possible_truncation, + reason = "filters in a single table tend to be a couple of megabytes of data at most, so easily fits into usize" + )] + let offset = reader.position() as usize; + + #[expect( + clippy::expect_used, + reason = "offset is expected to be with slice bounds" + )] + Ok(Self { + k, + m, + inner: BitArrayReader::new(slice.get(offset..).expect("should be in bounds")), + }) + } + + /// Size of bloom filter in bytes. + #[must_use] + #[expect(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.inner.bytes().len() + } + + /// Returns `true` if the hash may be contained. + /// + /// Will never have a false negative. + #[must_use] + pub fn contains_hash(&self, mut h1: u64) -> bool { + let mut h2 = secondary_hash(h1); + + for i in 1..=(self.k as u64) { + let idx = h1 % (self.m as u64); + + #[expect( + clippy::cast_possible_truncation, + reason = "filters in a single table tend to be a couple of megabytes of data at most, so easily fits into usize" + )] + if !self.has_bit(idx as usize) { + return false; + } + + h1 = h1.wrapping_add(h2); + h2 = h2.wrapping_mul(i); + } + + true + } + + /// Returns `true` if the item may be contained. + /// + /// Will never have a false negative. + #[must_use] + pub fn contains(&self, key: &[u8]) -> bool { + self.contains_hash(Self::get_hash(key)) + } + + /// Returns `true` if the bit at `idx` is `1`. + fn has_bit(&self, idx: usize) -> bool { + self.inner.get(idx) + } + + /// Gets the hash of a key. + fn get_hash(key: &[u8]) -> u64 { + Builder::get_hash(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn filter_bloom_standard_serde_round_trip() -> crate::Result<()> { + let mut filter = Builder::with_fp_rate(10, 0.0001); + + let keys = &[ + b"item0", b"item1", b"item2", b"item3", b"item4", b"item5", b"item6", b"item7", + b"item8", b"item9", + ]; + + for key in keys { + filter.set_with_hash(StandardBloomFilterReader::get_hash(*key)); + } + + let filter_bytes = filter.build(); + let filter_copy = StandardBloomFilterReader::new(&filter_bytes)?; + + assert_eq!(filter.k, filter_copy.k); + assert_eq!(filter.m, filter_copy.m); + assert!(!filter_copy.contains(b"asdasads")); + assert!(!filter_copy.contains(b"item10")); + assert!(!filter_copy.contains(b"cxycxycxy")); + + Ok(()) + } + + #[test] + fn filter_bloom_standard_basic() -> crate::Result<()> { + let mut filter = Builder::with_fp_rate(10, 0.0001); + + let keys = [ + b"item0" as &[u8], + b"item1", + b"item2", + b"item3", + b"item4", + b"item5", + b"item6", + b"item7", + b"item8", + b"item9", + ]; + + for key in &keys { + filter.set_with_hash(Builder::get_hash(key)); + } + + let filter_bytes = filter.build(); + let filter = StandardBloomFilterReader::new(&filter_bytes)?; + + for key in &keys { + assert!(filter.contains(key)); + } + + assert!(!filter.contains(b"asdasdasdasdasdasdasd")); + + Ok(()) + } + + #[test] + fn filter_bloom_standard_bpk() -> crate::Result<()> { + let item_count = 1_000; + let bpk = 5.0; + + let mut filter = Builder::with_bpk(item_count, bpk); + + for key in (0..item_count).map(|_| nanoid::nanoid!()) { + let key = key.as_bytes(); + + filter.set_with_hash(Builder::get_hash(key)); + } + + let filter_bytes = filter.build(); + let filter = StandardBloomFilterReader::new(&filter_bytes)?; + + let mut false_positives = 0; + + for key in (0..item_count).map(|_| nanoid::nanoid!()) { + let key = key.as_bytes(); + + if filter.contains(key) { + false_positives += 1; + } + } + + #[expect(clippy::cast_precision_loss)] + let fpr = false_positives as f32 / item_count as f32; + assert!(fpr < 0.13); + + Ok(()) + } + + #[test] + fn filter_bloom_standard_fpr() -> crate::Result<()> { + let item_count = 100_000; + let wanted_fpr = 0.1; + + let mut filter = Builder::with_fp_rate(item_count, wanted_fpr); + + for key in (0..item_count).map(|_| nanoid::nanoid!()) { + let key = key.as_bytes(); + + filter.set_with_hash(Builder::get_hash(key)); + } + + let filter_bytes = filter.build(); + let filter = StandardBloomFilterReader::new(&filter_bytes)?; + + let mut false_positives = 0; + + for key in (0..item_count).map(|_| nanoid::nanoid!()) { + let key = key.as_bytes(); + + if filter.contains(key) { + false_positives += 1; + } + } + + #[expect(clippy::cast_precision_loss)] + let fpr = false_positives as f32 / item_count as f32; + assert!(fpr > 0.05); + assert!(fpr < 0.13); + + Ok(()) + } + + #[test] + fn filter_bloom_standard_fpr_2() -> crate::Result<()> { + let item_count = 100_000; + let wanted_fpr = 0.5; + + let mut filter = Builder::with_fp_rate(item_count, wanted_fpr); + + for key in (0..item_count).map(|_| nanoid::nanoid!()) { + let key = key.as_bytes(); + + filter.set_with_hash(Builder::get_hash(key)); + } + + let filter_bytes = filter.build(); + let filter = StandardBloomFilterReader::new(&filter_bytes)?; + + let mut false_positives = 0; + + for key in (0..item_count).map(|_| nanoid::nanoid!()) { + let key = key.as_bytes(); + + if filter.contains(key) { + false_positives += 1; + } + } + + #[expect(clippy::cast_precision_loss)] + let fpr = false_positives as f32 / item_count as f32; + assert!(fpr > 0.45); + assert!(fpr < 0.55); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/id.rs b/crates/lsm-tree/src/table/id.rs new file mode 100644 index 000000000..309c5e3d8 --- /dev/null +++ b/crates/lsm-tree/src/table/id.rs @@ -0,0 +1,44 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::tree::inner::TreeId; + +pub type TableId = u64; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct GlobalTableId(TreeId, TableId); + +impl GlobalTableId { + #[must_use] + pub fn tree_id(&self) -> TreeId { + self.0 + } + + #[must_use] + pub fn table_id(&self) -> TableId { + self.1 + } +} + +impl From<(TreeId, TableId)> for GlobalTableId { + fn from((tid, sid): (TreeId, TableId)) -> Self { + Self(tid, sid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn global_table_id_accessors() { + let tree_id = 42; + let table_id: TableId = 7; + let global_table_id = GlobalTableId::from((tree_id, table_id)); + + assert_eq!(global_table_id.tree_id(), 42); + assert_eq!(global_table_id.table_id(), 7); + } +} diff --git a/crates/lsm-tree/src/table/index_block/block_handle.rs b/crates/lsm-tree/src/table/index_block/block_handle.rs new file mode 100644 index 000000000..bbc11f31f --- /dev/null +++ b/crates/lsm-tree/src/table/index_block/block_handle.rs @@ -0,0 +1,266 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{SeqNo, UserKey}; +use crate::{ + coding::{Decode, Encode}, + table::{ + block::{BlockOffset, Decodable, Encodable, TRAILER_START_MARKER}, + index_block::IndexBlockParsedItem, + util::SliceIndexes, + }, +}; +use byteorder::{ReadBytesExt, WriteBytesExt}; +use std::io::{Cursor, Seek}; +use varint_rs::{VarintReader, VarintWriter}; + +/// Points to a block on file +#[derive(Copy, Clone, Debug, Default)] +pub struct BlockHandle { + /// Position of block in file + offset: BlockOffset, + + /// Size of block in bytes + size: u32, +} + +impl BlockHandle { + #[must_use] + pub fn new(offset: BlockOffset, size: u32) -> Self { + Self { offset, size } + } + + #[must_use] + pub fn size(&self) -> u32 { + self.size + } + + #[must_use] + pub fn offset(&self) -> BlockOffset { + self.offset + } +} + +impl Encode for BlockHandle { + fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error> { + writer.write_u64_varint(*self.offset)?; + writer.write_u32_varint(self.size)?; + Ok(()) + } +} + +impl Decode for BlockHandle { + fn decode_from(reader: &mut R) -> Result + where + Self: Sized, + { + let offset = reader.read_u64_varint()?; + let size = reader.read_u32_varint()?; + + Ok(Self { + offset: BlockOffset(offset), + size, + }) + } +} + +/// Points to a block on file +#[derive(Clone, Debug)] +pub struct KeyedBlockHandle { + /// Key of last item in block + end_key: UserKey, + + /// Seqno of last item in block + seqno: SeqNo, + + inner: BlockHandle, +} + +impl AsRef for KeyedBlockHandle { + fn as_ref(&self) -> &BlockHandle { + &self.inner + } +} + +impl KeyedBlockHandle { + #[must_use] + pub fn into_inner(self) -> BlockHandle { + self.inner + } + + #[must_use] + pub fn new(end_key: UserKey, seqno: SeqNo, handle: BlockHandle) -> Self { + Self { + end_key, + seqno, + inner: handle, + } + } + + #[must_use] + pub fn seqno(&self) -> SeqNo { + self.seqno + } + + pub fn shift(&mut self, delta: BlockOffset) { + self.inner.offset += delta; + } + + #[must_use] + pub fn size(&self) -> u32 { + self.inner.size() + } + + #[must_use] + pub fn offset(&self) -> BlockOffset { + self.inner.offset() + } + + #[must_use] + pub fn end_key(&self) -> &UserKey { + &self.end_key + } +} + +#[cfg(test)] +impl PartialEq for KeyedBlockHandle { + fn eq(&self, other: &Self) -> bool { + self.offset() == other.offset() + } +} + +impl Encodable for KeyedBlockHandle { + fn encode_full_into( + &self, + writer: &mut W, + state: &mut BlockOffset, + _fixed_key_len: Option, + _fixed_value_len: Option, + ) -> crate::Result<()> { + // We encode restart markers as: + // [marker=0] [offset] [size] [seqno] [key len] [end key] + // 1 2 3 4 5 6 + + writer.write_u8(0)?; // 1 + + self.inner.encode_into(writer)?; // 2, 3 + + unwrap!(writer.write_u64_varint(self.seqno)); // 4 + + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 long max")] + writer.write_u16_varint(self.end_key.len() as u16)?; // 5 + writer.write_all(&self.end_key)?; // 6 + + *state = BlockOffset(*self.offset() + u64::from(self.size())); + + Ok(()) + } + + // TODO: see https://github.com/fjall-rs/lsm-tree/issues/184 + #[cfg_attr(coverage_nightly, coverage(off))] + fn encode_truncated_into( + &self, + _writer: &mut W, + _state: &mut BlockOffset, + _shared_len: usize, + _fixed_key_len: Option, + _fixed_value_len: Option, + ) -> crate::Result<()> { + unimplemented!() + } + + fn key(&self) -> &[u8] { + &self.end_key + } +} + +impl Decodable for KeyedBlockHandle { + fn parse_full( + reader: &mut Cursor<&[u8]>, + offset: usize, + fixed_key_len: Option, + _fixed_value_len: Option, + ) -> Option { + let marker = unwrap!(reader.read_u8()); + + if marker == TRAILER_START_MARKER { + return None; + } + + let handle = unwrap!(BlockHandle::decode_from(reader)); + let seqno = unwrap!(reader.read_u64_varint()); + + let key_len: usize = + fixed_key_len.map_or_else(|| unwrap!(reader.read_u16_varint()).into(), usize::from); + #[expect( + clippy::cast_possible_truncation, + reason = "blocks tend to be some megabytes in size at most, so position should fit into usize" + )] + let key_start = offset + reader.position() as usize; + + #[expect( + clippy::cast_possible_wrap, + reason = "key_len is bounded by u16::MAX, no wrap expected" + )] + let offset_i64 = key_len as i64; + unwrap!(reader.seek_relative(offset_i64)); + + Some(IndexBlockParsedItem { + prefix: None, + end_key: SliceIndexes(key_start, key_start + key_len), + offset: handle.offset(), + size: handle.size(), + seqno, + }) + } + + fn parse_restart_key<'a>( + reader: &mut Cursor<&[u8]>, + offset: usize, + data: &'a [u8], + fixed_key_len: Option, + _fixed_value_len: Option, + ) -> Option<(&'a [u8], SeqNo)> { + let marker = unwrap!(reader.read_u8()); + + if marker == TRAILER_START_MARKER { + return None; + } + + let _file_offset = unwrap!(reader.read_u64_varint()); + let _size = unwrap!(reader.read_u32_varint()); + let seqno = unwrap!(reader.read_u64_varint()); + + let key_len: usize = + fixed_key_len.map_or_else(|| unwrap!(reader.read_u16_varint()).into(), usize::from); + #[expect( + clippy::cast_possible_truncation, + reason = "blocks tend to be some megabytes in size at most, so position should fit into usize" + )] + let key_start = offset + reader.position() as usize; + + #[expect( + clippy::cast_possible_wrap, + reason = "key_len is bounded by u16::MAX, no wrap expected" + )] + let key_len_i64 = key_len as i64; + unwrap!(reader.seek_relative(key_len_i64)); + + data.get(key_start..) + .and_then(|s| s.get(..key_len)) + .map(|k| (k, seqno)) + } + + // TODO: see https://github.com/fjall-rs/lsm-tree/issues/184 + #[cfg_attr(coverage_nightly, coverage(off))] + fn parse_truncated( + _reader: &mut Cursor<&[u8]>, + _offset: usize, + _base_key_offset: usize, + _fixed_key_len: Option, + _fixed_value_len: Option, + ) -> Option { + unimplemented!() + } +} diff --git a/crates/lsm-tree/src/table/index_block/iter.rs b/crates/lsm-tree/src/table/index_block/iter.rs new file mode 100644 index 000000000..1ef9dd2b9 --- /dev/null +++ b/crates/lsm-tree/src/table/index_block/iter.rs @@ -0,0 +1,672 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + SeqNo, + double_ended_peekable::{DoubleEndedPeekable, DoubleEndedPeekableExt}, + table::{KeyedBlockHandle, block::Decoder, index_block::IndexBlockParsedItem}, +}; + +pub struct Iter<'a> { + decoder: DoubleEndedPeekable< + IndexBlockParsedItem, + Decoder<'a, KeyedBlockHandle, IndexBlockParsedItem>, + >, +} + +impl<'a> Iter<'a> { + #[must_use] + pub fn new(decoder: Decoder<'a, KeyedBlockHandle, IndexBlockParsedItem>) -> Self { + let decoder = decoder.double_ended_peekable(); + Self { decoder } + } + + pub fn seek(&mut self, needle: &[u8], seqno: SeqNo) -> bool { + self.decoder.inner_mut().seek( + |end_key, s| match end_key.cmp(needle) { + std::cmp::Ordering::Greater => false, + std::cmp::Ordering::Less => true, + std::cmp::Ordering::Equal => s >= seqno, + }, + true, + ) + } + + pub fn seek_upper(&mut self, needle: &[u8], _seqno: SeqNo) -> bool { + self.decoder + .inner_mut() + .seek_upper(|end_key, _s| end_key <= needle, true) + } +} + +impl Iterator for Iter<'_> { + type Item = IndexBlockParsedItem; + + fn next(&mut self) -> Option { + self.decoder.next() + } +} + +impl DoubleEndedIterator for Iter<'_> { + fn next_back(&mut self) -> Option { + self.decoder.next_back() + } +} + +#[cfg(test)] +mod tests { + use crate::{ + Checksum, + table::{ + Block, BlockHandle, BlockOffset, IndexBlock, KeyedBlockHandle, + block::{BlockType, Header, ParsedItem}, + }, + }; + use test_log::test; + + #[test] + fn index_block_iter_seek_before_start() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 0), "should seek"); + + let iter = index_block + .iter() + .map(|item| item.materialize(&index_block.inner.data)); + + let real_items: Vec<_> = iter.collect(); + + assert_eq!(items, &*real_items); + + Ok(()) + } + + #[test] + fn index_block_iter_seek_start() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(iter.seek(b"b", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items, &*real_items); + + Ok(()) + } + + #[test] + fn index_block_iter_seek_middle() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(iter.seek(b"c", 0), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(2).cloned().collect::>(), + &*real_items, + ); + + Ok(()) + } + + #[test] + fn index_block_iter_rev_seek() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(iter.seek_upper(b"c", 0), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items.to_vec(), &*real_items); + + Ok(()) + } + + #[test] + fn index_block_iter_rev_seek_2() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(iter.seek_upper(b"e", 0), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items.to_vec(), &*real_items); + + Ok(()) + } + + #[test] + fn index_block_iter_rev_seek_3() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(iter.seek_upper(b"b", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().take(2).cloned().collect::>(), + &*real_items, + ); + + Ok(()) + } + + #[test] + fn index_block_iter_too_far() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + let mut iter = index_block.iter(); + assert!(!iter.seek(b"zzz", 0), "should not seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(&[] as &[KeyedBlockHandle], &*real_items); + + Ok(()) + } + + #[test] + fn index_block_iter_too_far_next_back() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new( + b"bcdef".into(), + 0, + BlockHandle::new(BlockOffset(6_000), 7_000), + ), + KeyedBlockHandle::new( + b"def".into(), + 0, + BlockHandle::new(BlockOffset(13_000), 5_000), + ), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + let mut iter = index_block.iter(); + assert!(!iter.seek(b"zzz", 0), "should not seek"); + + assert!(iter.next().is_none(), "iterator should be exhausted"); + assert!( + iter.next_back().is_none(), + "reverse iterator should also be exhausted" + ); + + Ok(()) + } + + #[test] + fn index_block_mvcc_slab() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"a".into(), 3, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new(b"a".into(), 1, BlockHandle::new(BlockOffset(6_000), 7_000)), + KeyedBlockHandle::new(b"b".into(), 4, BlockHandle::new(BlockOffset(13_000), 5_000)), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 5), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items, &*real_items); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 4), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items, &*real_items); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 3), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(1).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 2), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(1).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(2).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 0), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(2).cloned().collect::>(), + &*real_items, + ); + } + + Ok(()) + } + + #[test] + fn index_block_iter_span() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"a".into(), 1, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new(b"a".into(), 0, BlockHandle::new(BlockOffset(6_000), 7_000)), + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(13_000), 5_000)), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"a", 2), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items.to_vec(), &*real_items); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"b", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(2).cloned().collect::>(), + &*real_items, + ); + } + + Ok(()) + } + + #[test] + fn index_block_iter_rev_span() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"a".into(), 1, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new(b"a".into(), 0, BlockHandle::new(BlockOffset(6_000), 7_000)), + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(13_000), 5_000)), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + { + let mut iter = index_block.iter(); + assert!(iter.seek_upper(b"a", 2), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items.to_vec(), &*real_items); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek_upper(b"b", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!(items.to_vec(), &*real_items); + } + + Ok(()) + } + + #[test] + fn index_block_iter_range_1() -> crate::Result<()> { + let items = [ + KeyedBlockHandle::new(b"a".into(), 0, BlockHandle::new(BlockOffset(0), 6_000)), + KeyedBlockHandle::new(b"b".into(), 0, BlockHandle::new(BlockOffset(13_000), 5_000)), + KeyedBlockHandle::new(b"c".into(), 0, BlockHandle::new(BlockOffset(13_000), 5_000)), + KeyedBlockHandle::new(b"d".into(), 0, BlockHandle::new(BlockOffset(13_000), 5_000)), + KeyedBlockHandle::new(b"e".into(), 0, BlockHandle::new(BlockOffset(13_000), 5_000)), + ]; + + let bytes = IndexBlock::encode_into_vec(&items)?; + + let index_block = IndexBlock::new(Block { + data: bytes.into(), + header: Header { + block_type: BlockType::Index, + checksum: Checksum::from_raw(0), + data_length: 0, + uncompressed_length: 0, + }, + }); + + assert_eq!(index_block.len(), items.len()); + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"b", 1), "should seek"); + assert!(iter.seek_upper(b"c", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items.iter().skip(1).take(3).cloned().collect::>(), + &*real_items, + ); + } + + { + let mut iter = index_block.iter(); + assert!(iter.seek(b"b", 1), "should seek"); + assert!(iter.seek_upper(b"c", 1), "should seek"); + + let real_items: Vec<_> = iter + .map(|item| item.materialize(&index_block.inner.data)) + .collect(); + + assert_eq!( + items + .iter() + .skip(1) + .take(3) + .rev() + .cloned() + .collect::>(), + &*real_items, + ); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/index_block/mod.rs b/crates/lsm-tree/src/table/index_block/mod.rs new file mode 100644 index 000000000..5f8d4610b --- /dev/null +++ b/crates/lsm-tree/src/table/index_block/mod.rs @@ -0,0 +1,128 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod block_handle; +mod iter; + +pub use block_handle::{BlockHandle, KeyedBlockHandle}; +pub use iter::Iter; + +use super::{ + Block, + block::{BlockOffset, Encoder, Trailer}, +}; +use crate::Slice; +use crate::{ + SeqNo, + table::{ + block::{Decoder, ParsedItem}, + util::{SliceIndexes, compare_prefixed_slice}, + }, +}; + +#[derive(Debug)] +pub struct IndexBlockParsedItem { + pub offset: BlockOffset, + pub size: u32, + pub prefix: Option, + pub end_key: SliceIndexes, + pub seqno: SeqNo, +} + +impl ParsedItem for IndexBlockParsedItem { + fn compare_key(&self, needle: &[u8], bytes: &[u8]) -> std::cmp::Ordering { + if let Some(prefix) = &self.prefix { + let prefix = unsafe { bytes.get_unchecked(prefix.0..prefix.1) }; + let rest_key = unsafe { bytes.get_unchecked(self.end_key.0..self.end_key.1) }; + compare_prefixed_slice(prefix, rest_key, needle) + } else { + let key = unsafe { bytes.get_unchecked(self.end_key.0..self.end_key.1) }; + key.cmp(needle) + } + } + + fn key_offset(&self) -> usize { + self.end_key.0 + } + + fn materialize(&self, bytes: &Slice) -> KeyedBlockHandle { + // NOTE: We consider the prefix and key slice indexes to be trustworthy + #[expect(clippy::indexing_slicing)] + let key = if let Some(prefix) = &self.prefix { + let prefix_key = &bytes[prefix.0..prefix.1]; + let rest_key = &bytes[self.end_key.0..self.end_key.1]; + Slice::fused(prefix_key, rest_key) + } else { + bytes.slice(self.end_key.0..self.end_key.1) + }; + + KeyedBlockHandle::new(key, self.seqno, BlockHandle::new(self.offset, self.size)) + } +} + +/// Block that contains block handles (file offset + size) +#[derive(Clone)] +pub struct IndexBlock { + pub inner: Block, +} + +impl IndexBlock { + #[must_use] + pub fn new(inner: Block) -> Self { + Self { inner } + } + + /// Accesses the inner raw bytes + #[must_use] + pub fn as_slice(&self) -> &Slice { + &self.inner.data + } + + /// Returns the number of items in the block. + #[must_use] + #[expect(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + Trailer::new(&self.inner).item_count() + } + + #[must_use] + #[expect(clippy::iter_without_into_iter)] + pub fn iter(&self) -> Iter<'_> { + Iter::new(Decoder::::new( + &self.inner, + )) + } + + pub fn encode_into_vec(items: &[KeyedBlockHandle]) -> crate::Result> { + let mut buf = vec![]; + + Self::encode_into(&mut buf, items)?; + + Ok(buf) + } + + /// Builds an index block. + /// + /// # Panics + /// + /// Panics if the given item array if empty. + pub fn encode_into(writer: &mut Vec, items: &[KeyedBlockHandle]) -> crate::Result<()> { + #[expect(clippy::expect_used)] + let first_key = items.first().expect("chunk should not be empty").end_key(); + + let mut serializer = Encoder::<'_, BlockOffset, KeyedBlockHandle>::new( + writer, + items.len(), + 1, // hard-coded for now, TODO: see https://github.com/fjall-rs/lsm-tree/issues/184 + 0.0, // Index blocks do not support hash index + first_key, + ); + + for item in items { + serializer.write(item)?; + } + + serializer.finish() + } +} diff --git a/crates/lsm-tree/src/table/inner.rs b/crates/lsm-tree/src/table/inner.rs new file mode 100644 index 000000000..749768e48 --- /dev/null +++ b/crates/lsm-tree/src/table/inner.rs @@ -0,0 +1,97 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +use super::{block_index::BlockIndexImpl, meta::ParsedMeta, regions::ParsedRegions}; +use crate::{ + Checksum, GlobalTableId, SeqNo, + cache::Cache, + file_accessor::FileAccessor, + table::{IndexBlock, filter::block::FilterBlock}, + tree::inner::TreeId, +}; +use std::{ + path::PathBuf, + sync::{Arc, OnceLock, atomic::AtomicBool}, +}; + +pub struct Inner { + pub path: Arc, + + pub(crate) tree_id: TreeId, + + #[doc(hidden)] + pub(crate) file_accessor: FileAccessor, + + /// Parsed metadata + #[doc(hidden)] + pub metadata: ParsedMeta, + + /// Parsed region block handles + #[doc(hidden)] + pub regions: ParsedRegions, + + /// Translates key (first item of a block) to block offset (address inside file) and (compressed) size + #[doc(hidden)] + pub block_index: Arc, + + /// Block cache + /// + /// Stores index and data blocks + #[doc(hidden)] + pub cache: Arc, + + /// Pinned filter index (in case of partitioned filters) + pub(super) pinned_filter_index: Option, + + /// Pinned AMQ filter + pub pinned_filter_block: Option, + + /// True when the table was compacted away or dropped + /// + /// May be kept alive until all Arcs to the table have been dropped (to facilitate snapshots) + pub is_deleted: AtomicBool, + + pub(super) checksum: Checksum, + + pub(super) global_seqno: SeqNo, + + #[cfg(feature = "metrics")] + pub(crate) metrics: Arc, + + /// Cached sum of referenced blob file bytes for this table. + /// Lazily computed on first access to avoid repeated I/O in compaction decisions. + pub(crate) cached_blob_bytes: OnceLock, +} + +impl Inner { + /// Gets the global table ID. + #[must_use] + pub(super) fn global_id(&self) -> GlobalTableId { + (self.tree_id, self.metadata.id).into() + } +} + +impl Drop for Inner { + fn drop(&mut self) { + let global_id = self.global_id(); + + if self.is_deleted.load(std::sync::atomic::Ordering::Acquire) { + log::trace!("Cleanup deleted table {global_id:?} at {:?}", self.path); + + if let Err(e) = std::fs::remove_file(&*self.path) { + log::warn!( + "Failed to cleanup deleted table {global_id:?} at {:?}: {e:?}", + self.path, + ); + } + + self.file_accessor.as_descriptor_table().inspect(|d| { + d.remove_for_table(&global_id); + }); + } + } +} diff --git a/crates/lsm-tree/src/table/iter.rs b/crates/lsm-tree/src/table/iter.rs new file mode 100644 index 000000000..1eb9c0193 --- /dev/null +++ b/crates/lsm-tree/src/table/iter.rs @@ -0,0 +1,414 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{BlockOffset, DataBlock, GlobalTableId, data_block::Iter as DataBlockIter}; +use crate::{ + Cache, CompressionType, InternalValue, SeqNo, UserKey, + file_accessor::FileAccessor, + table::{ + BlockHandle, + block::ParsedItem, + block_index::{BlockIndexIter, BlockIndexIterImpl}, + util::load_block, + }, +}; +use self_cell::self_cell; +use std::{path::PathBuf, sync::Arc}; + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +type InnerIter<'a> = DataBlockIter<'a>; + +pub enum Bound { + Included(UserKey), + Excluded(UserKey), +} +type Bounds = (Option, Option); + +self_cell!( + pub struct OwnedDataBlockIter { + owner: DataBlock, + + #[covariant] + dependent: InnerIter, + } +); + +impl OwnedDataBlockIter { + fn seek_lower_inclusive(&mut self, needle: &[u8], _seqno: SeqNo) -> bool { + self.with_dependent_mut(|_, m| m.seek(needle /* TODO: , seqno */)) + } + + fn seek_upper_inclusive(&mut self, needle: &[u8], _seqno: SeqNo) -> bool { + self.with_dependent_mut(|_, m| m.seek_upper(needle /* TODO: , seqno */)) + } + + fn seek_lower_exclusive(&mut self, needle: &[u8], _seqno: SeqNo) -> bool { + self.with_dependent_mut(|_, m| m.seek_exclusive(needle /* TODO: , seqno */)) + } + + fn seek_upper_exclusive(&mut self, needle: &[u8], _seqno: SeqNo) -> bool { + self.with_dependent_mut(|_, m| m.seek_upper_exclusive(needle /* TODO: , seqno */)) + } + + pub fn seek_lower_bound(&mut self, bound: &Bound, seqno: SeqNo) -> bool { + match bound { + Bound::Included(key) => self.seek_lower_inclusive(key, seqno), + Bound::Excluded(key) => self.seek_lower_exclusive(key, seqno), + } + } + + pub fn seek_upper_bound(&mut self, bound: &Bound, seqno: SeqNo) -> bool { + match bound { + Bound::Included(key) => self.seek_upper_inclusive(key, seqno), + Bound::Excluded(key) => self.seek_upper_exclusive(key, seqno), + } + } +} + +impl Iterator for OwnedDataBlockIter { + type Item = InternalValue; + + fn next(&mut self) -> Option { + self.with_dependent_mut(|block, iter| { + iter.next().map(|item| item.materialize(&block.inner.data)) + }) + } +} + +impl DoubleEndedIterator for OwnedDataBlockIter { + fn next_back(&mut self) -> Option { + self.with_dependent_mut(|block, iter| { + iter.next_back() + .map(|item| item.materialize(&block.inner.data)) + }) + } +} + +fn create_data_block_reader(block: DataBlock) -> OwnedDataBlockIter { + OwnedDataBlockIter::new(block, super::data_block::DataBlock::iter) +} + +pub struct Iter { + table_id: GlobalTableId, + path: Arc, + + global_seqno: SeqNo, + + #[expect(clippy::struct_field_names)] + index_iter: BlockIndexIterImpl, + + file_accessor: FileAccessor, + cache: Arc, + compression: CompressionType, + + index_initialized: bool, + + lo_offset: BlockOffset, + lo_data_block: Option, + + hi_offset: BlockOffset, + hi_data_block: Option, + + range: Bounds, + + #[cfg(feature = "metrics")] + metrics: Arc, +} + +impl Iter { + pub fn new( + table_id: GlobalTableId, + global_seqno: SeqNo, + path: Arc, + index_iter: BlockIndexIterImpl, + file_accessor: FileAccessor, + cache: Arc, + compression: CompressionType, + #[cfg(feature = "metrics")] metrics: Arc, + ) -> Self { + Self { + table_id, + path, + + global_seqno, + + index_iter, + file_accessor, + cache, + compression, + + index_initialized: false, + + lo_offset: BlockOffset(0), + lo_data_block: None, + + hi_offset: BlockOffset(u64::MAX), + hi_data_block: None, + + range: (None, None), + + #[cfg(feature = "metrics")] + metrics, + } + } + + pub fn set_lower_bound(&mut self, bound: Bound) { + self.range.0 = Some(bound); + } + + pub fn set_upper_bound(&mut self, bound: Bound) { + self.range.1 = Some(bound); + } +} + +impl Iterator for Iter { + type Item = crate::Result; + + fn next(&mut self) -> Option { + // Always try to keep iterating inside the already-materialized low data block first; this + // lets callers consume multiple entries without touching the index or cache again. + if let Some(block) = &mut self.lo_data_block { + if let Some(item) = block + .next() + .map(|mut v| { + v.key.seqno += self.global_seqno; + v + }) + .map(Ok) + { + return Some(item); + } + } + + if !self.index_initialized { + // Lazily initialize the index iterator here (not in `new`) so callers can set bounds + // before we incur any seek or I/O cost. Bounds exclusivity is enforced at the data- + // block level; index seeks only narrow the span of blocks to touch. + let mut ok = if let Some(bound) = &self.range.0 { + // Seek to the first block whose end key is ≥ lower bound. + // If this fails we can immediately conclude the range is empty. + let key = match bound { + Bound::Included(k) | Bound::Excluded(k) => k, + }; + self.index_iter.seek_lower(key, u64::MAX) + } else { + true + }; + + if ok { + // Apply an upper-bound seek to cap the block span, but keep exact high-key + // handling inside the data block so exclusivity is respected precisely. + if let Some(bound) = &self.range.1 { + let key = match bound { + Bound::Included(k) | Bound::Excluded(k) => k, + }; + ok = self.index_iter.seek_upper(key, u64::MAX); + } + } + + self.index_initialized = true; + + if !ok { + // No block in the index overlaps the requested window, so we clear state and return + // EOF without attempting to touch any data blocks. + self.lo_data_block = None; + self.hi_data_block = None; + return None; + } + } + + loop { + let Some(handle) = self.index_iter.next() else { + // No more block handles coming from the index. Flush any pending items buffered on + // the high side (used by reverse iteration) before signalling completion. + if let Some(block) = &mut self.hi_data_block { + if let Some(item) = block + .next() + .map(|mut v| { + v.key.seqno += self.global_seqno; + v + }) + .map(Ok) + { + return Some(item); + } + } + + // Nothing left to serve; drop both buffers so the iterator can be reused safely. + self.lo_data_block = None; + self.hi_data_block = None; + return None; + }; + let handle = fail_iter!(handle); + + // Load the next data block referenced by the index handle. We try the shared block + // cache first to avoid hitting the filesystem, and fall back to `load_block` on miss. + let block = match self.cache.get_block(self.table_id, handle.offset()) { + Some(block) => block, + None => { + fail_iter!(load_block( + self.table_id, + &self.path, + &self.file_accessor, + &self.cache, + &BlockHandle::new(handle.offset(), handle.size()), + crate::table::block::BlockType::Data, + self.compression, + #[cfg(feature = "metrics")] + &self.metrics, + )) + } + }; + let block = DataBlock::new(block); + + let mut reader = create_data_block_reader(block); + + // Forward path: seek the low side first to avoid returning entries below the lower + // bound, then clamp the iterator on the high side. This guarantees iteration stays in + // [low, high] with exact control over inclusivity/exclusivity. + if let Some(bound) = &self.range.0 { + reader.seek_lower_bound(bound, SeqNo::MAX); + } + if let Some(bound) = &self.range.1 { + reader.seek_upper_bound(bound, SeqNo::MAX); + } + + let item = reader.next(); + + self.lo_offset = handle.offset(); + self.lo_data_block = Some(reader); + + if let Some(mut item) = item { + item.key.seqno += self.global_seqno; + + // Serving the first item immediately avoids stashing it in a temporary buffer and + // keeps block iteration semantics identical to the simple case at the top. + return Some(Ok(item)); + } + } + } +} + +impl DoubleEndedIterator for Iter { + fn next_back(&mut self) -> Option { + // Mirror the forward iterator: prefer consuming buffered items from the high data block to + // avoid touching the index once a block has been materialized. + if let Some(block) = &mut self.hi_data_block { + if let Some(item) = block + .next_back() + .map(|mut v| { + v.key.seqno += self.global_seqno; + v + }) + .map(Ok) + { + return Some(item); + } + } + + if !self.index_initialized { + // Mirror forward iteration: initialize lazily so bounds can be applied up-front. The + // index only restricts which blocks we consider; tight bound enforcement happens in + // the data block readers below. + let mut ok = if let Some(bound) = &self.range.0 { + let key = match bound { + Bound::Included(k) | Bound::Excluded(k) => k, + }; + self.index_iter.seek_lower(key, u64::MAX) + } else { + true + }; + + if ok { + if let Some(bound) = &self.range.1 { + let key = match bound { + Bound::Included(k) | Bound::Excluded(k) => k, + }; + ok = self.index_iter.seek_upper(key, u64::MAX); + } + } + + self.index_initialized = true; + + if !ok { + // No index span overlaps the requested window; clear both buffers and finish early. + self.lo_data_block = None; + self.hi_data_block = None; + return None; + } + } + + loop { + let Some(handle) = self.index_iter.next_back() else { + // Once we exhaust the index in reverse order, flush any items that were buffered on + // the low side (set when iterating forward first) before signalling completion. + if let Some(block) = &mut self.lo_data_block { + if let Some(item) = block + .next_back() + .map(|mut v| { + v.key.seqno += self.global_seqno; + v + }) + .map(Ok) + { + return Some(item); + } + } + + // Nothing left to produce; reset both buffers to keep the iterator reusable. + self.lo_data_block = None; + self.hi_data_block = None; + return None; + }; + let handle = fail_iter!(handle); + + // Retrieve the next data block from the cache (or disk on miss) so the high-side reader + // can serve entries in reverse order. + let block = match self.cache.get_block(self.table_id, handle.offset()) { + Some(block) => block, + None => { + fail_iter!(load_block( + self.table_id, + &self.path, + &self.file_accessor, + &self.cache, + &BlockHandle::new(handle.offset(), handle.size()), + crate::table::block::BlockType::Data, + self.compression, + #[cfg(feature = "metrics")] + &self.metrics, + )) + } + }; + let block = DataBlock::new(block); + + let mut reader = create_data_block_reader(block); + + // Reverse path: clamp the high side first so `next_back` never yields an entry above + // the upper bound, then apply the low-side seek to avoid stepping below the lower + // bound during reverse traversal. + if let Some(bound) = &self.range.1 { + reader.seek_upper_bound(bound, SeqNo::MAX); + } + if let Some(bound) = &self.range.0 { + reader.seek_lower_bound(bound, SeqNo::MAX); + } + + let item = reader.next_back(); + + self.hi_offset = handle.offset(); + self.hi_data_block = Some(reader); + + if let Some(mut item) = item { + item.key.seqno += self.global_seqno; + + // Emit the first materialized entry immediately to match the forward path and avoid + // storing it in a temporary buffer. + return Some(Ok(item)); + } + } + } +} diff --git a/crates/lsm-tree/src/table/meta.rs b/crates/lsm-tree/src/table/meta.rs new file mode 100644 index 000000000..946c855ef --- /dev/null +++ b/crates/lsm-tree/src/table/meta.rs @@ -0,0 +1,225 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Block, BlockHandle, DataBlock}; +use crate::{ + CompressionType, KeyRange, SeqNo, TableId, checksum::ChecksumType, coding::Decode, + table::block::BlockType, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::{fs::File, ops::Deref}; + +/// Nanosecond timestamp. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd)] +pub struct Timestamp(u128); + +impl Deref for Timestamp { + type Target = u128; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for u128 { + fn from(val: Timestamp) -> Self { + val.0 + } +} + +impl From for Timestamp { + fn from(value: u128) -> Self { + Self(value) + } +} + +#[derive(Debug)] +pub struct ParsedMeta { + pub id: TableId, + pub created_at: Timestamp, + pub data_block_count: u64, + pub index_block_count: u64, + pub key_range: KeyRange, + pub(super) seqnos: (SeqNo, SeqNo), + pub file_size: u64, + pub item_count: u64, + pub tombstone_count: u64, + pub weak_tombstone_count: u64, + pub weak_tombstone_reclaimable: u64, + + pub data_block_compression: CompressionType, + pub index_block_compression: CompressionType, +} + +macro_rules! read_u8 { + ($block:expr, $name:expr) => {{ + let bytes = $block + .point_read($name, SeqNo::MAX) + .unwrap_or_else(|| panic!("meta property {:?} should exist", $name)); + + let mut bytes = &bytes.value[..]; + bytes.read_u8()? + }}; +} + +macro_rules! read_u64 { + ($block:expr, $name:expr) => {{ + let bytes = $block + .point_read($name, SeqNo::MAX) + .unwrap_or_else(|| panic!("meta property {:?} should exist", $name)); + + let mut bytes = &bytes.value[..]; + bytes.read_u64::()? + }}; +} + +impl ParsedMeta { + #[expect(clippy::expect_used, clippy::too_many_lines)] + pub fn load_with_handle(file: &File, handle: &BlockHandle) -> crate::Result { + let block = Block::from_file(file, *handle, CompressionType::None)?; + + if block.header.block_type != BlockType::Meta { + return Err(crate::Error::InvalidTag(( + "BlockType", + block.header.block_type.into(), + ))); + } + + let block = DataBlock::new(block); + + #[expect(clippy::indexing_slicing)] + { + let table_version = block + .point_read(b"table_version", SeqNo::MAX) + .expect("Table version should exist") + .value; + + assert!( + matches!(&*table_version, [3u8 | 4u8]), + "unsupported table version {}", + table_version[0], + ); + } + + { + let hash_type = block + .point_read(b"filter_hash_type", SeqNo::MAX) + .expect("Filter hash type should exist") + .value; + + assert_eq!( + &[u8::from(ChecksumType::Xxh3)], + &*hash_type, + "invalid hash type: {:?}", + std::str::from_utf8(&hash_type), + ); + } + + { + let hash_type = block + .point_read(b"checksum_type", SeqNo::MAX) + .expect("Checksum type should exist") + .value; + + assert_eq!( + &[u8::from(ChecksumType::Xxh3)], + &*hash_type, + "invalid checksum type: {:?}", + std::str::from_utf8(&hash_type), + ); + } + + assert_eq!( + read_u8!(block, b"restart_interval#index"), + 1, + "index block restart intervals >1 are not supported for this version", + ); + + let id = read_u64!(block, b"table_id"); + let item_count = read_u64!(block, b"item_count"); + let tombstone_count = read_u64!(block, b"tombstone_count"); + let data_block_count = read_u64!(block, b"block_count#data"); + let index_block_count = read_u64!(block, b"block_count#index"); + let _filter_block_count = read_u64!(block, b"block_count#filter"); + let file_size = read_u64!(block, b"file_size"); + let weak_tombstone_count = read_u64!(block, b"weak_tombstone_count"); + let weak_tombstone_reclaimable = read_u64!(block, b"weak_tombstone_reclaimable"); + + let created_at = { + let bytes = block + .point_read(b"created_at", SeqNo::MAX) + .expect("created_at timestamp should exist"); + + let mut bytes = &bytes.value[..]; + bytes.read_u128::()?.into() + }; + + let key_range = KeyRange::new(( + block + .point_read(b"key#min", SeqNo::MAX) + .expect("key min should exist") + .value, + block + .point_read(b"key#max", SeqNo::MAX) + .expect("key max should exist") + .value, + )); + + let seqnos = { + let min = { + let bytes = block + .point_read(b"seqno#min", SeqNo::MAX) + .expect("seqno min should exist") + .value; + let mut bytes = &bytes[..]; + bytes.read_u64::()? + }; + + let max = { + let bytes = block + .point_read(b"seqno#max", SeqNo::MAX) + .expect("seqno max should exist") + .value; + let mut bytes = &bytes[..]; + bytes.read_u64::()? + }; + + (min, max) + }; + + let data_block_compression = { + let bytes = block + .point_read(b"compression#data", SeqNo::MAX) + .expect("size should exist"); + + let mut bytes = &bytes.value[..]; + CompressionType::decode_from(&mut bytes)? + }; + + let index_block_compression = { + let bytes = block + .point_read(b"compression#index", SeqNo::MAX) + .expect("size should exist"); + + let mut bytes = &bytes.value[..]; + CompressionType::decode_from(&mut bytes)? + }; + + Ok(Self { + id, + created_at, + data_block_count, + index_block_count, + key_range, + seqnos, + file_size, + item_count, + tombstone_count, + weak_tombstone_count, + weak_tombstone_reclaimable, + data_block_compression, + index_block_compression, + }) + } +} diff --git a/crates/lsm-tree/src/table/mod.rs b/crates/lsm-tree/src/table/mod.rs new file mode 100644 index 000000000..38da2de39 --- /dev/null +++ b/crates/lsm-tree/src/table/mod.rs @@ -0,0 +1,686 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod block; +pub(crate) mod block_index; +pub mod data_block; +pub mod filter; +mod id; +mod index_block; +mod inner; +mod iter; +mod meta; +pub(crate) mod multi_writer; +mod regions; +mod scanner; +pub mod util; +pub mod writer; + +#[cfg(test)] +mod tests; + +pub use block::{Block, BlockOffset}; +pub use data_block::DataBlock; +pub use id::{GlobalTableId, TableId}; +pub use index_block::{BlockHandle, IndexBlock, KeyedBlockHandle}; +pub use scanner::Scanner; +pub use writer::Writer; + +use crate::{ + Checksum, CompressionType, InternalValue, SeqNo, TreeId, UserKey, + cache::Cache, + descriptor_table::DescriptorTable, + file_accessor::FileAccessor, + table::{ + block::{BlockType, ParsedItem}, + block_index::{BlockIndex, FullBlockIndex, TwoLevelBlockIndex, VolatileBlockIndex}, + filter::block::FilterBlock, + regions::ParsedRegions, + writer::LinkedFile, + }, + value::PointReadValue, +}; +use block_index::BlockIndexImpl; +use inner::Inner; +use iter::Iter; +use std::{ + borrow::Cow, + fs::File, + ops::{Bound, RangeBounds}, + path::PathBuf, + sync::Arc, +}; +use util::load_block; + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +pub type TableInner = Inner; + +/// A disk segment (a.k.a. `Table`, `SSTable`, `SST`, `sorted string table`) that is located on disk +/// +/// A table is an immutable list of key-value pairs, split into compressed blocks. +/// A reference to the block (`block handle`) is saved in the "block index". +/// +/// Deleted entries are represented by tombstones. +/// +/// Tables can be merged together to improve read performance and free unneeded disk space by removing outdated item versions. +#[doc(alias("sstable", "sst", "sorted string table"))] +#[derive(Clone)] +pub struct Table(Arc); + +impl std::ops::Deref for Table { + type Target = Inner; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +impl std::fmt::Debug for Table { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Table:{}({:?})", self.id(), self.metadata.key_range) + } +} + +impl Table { + #[must_use] + pub fn global_seqno(&self) -> SeqNo { + self.0.global_seqno + } + + pub fn referenced_blob_bytes(&self) -> crate::Result { + if let Some(v) = self.0.cached_blob_bytes.get() { + return Ok(*v); + } + + let sum = self + .list_blob_file_references()? + .map(|bf| bf.iter().map(|f| f.on_disk_bytes).sum::()) + .unwrap_or_default(); + + let _ = self.0.cached_blob_bytes.set(sum); + Ok(sum) + } + + pub fn list_blob_file_references(&self) -> crate::Result>> { + use byteorder::{LE, ReadBytesExt}; + + Ok(if let Some(handle) = &self.regions.linked_blob_files { + let table_id = self.global_id(); + + let (fd, fd_cache_miss) = + if let Some(fd) = self.file_accessor.access_for_table(&table_id) { + (fd, false) + } else { + (Arc::new(File::open(&*self.path)?), true) + }; + + // Read the exact region using pread-style helper + let buf = crate::file::read_exact(&fd, *handle.offset(), handle.size() as usize)?; + + // If we opened the file here, cache the FD for future accesses + if fd_cache_miss { + self.file_accessor.insert_for_table(table_id, fd); + } + + // Parse the buffer + let mut reader = &buf[..]; + let len = reader.read_u32::()?; + let mut blob_files = Vec::with_capacity(len as usize); + + for _ in 0..len { + let blob_file_id = reader.read_u64::()?; + let len = reader.read_u64::()?; + let bytes = reader.read_u64::()?; + let on_disk_bytes = reader.read_u64::()?; + + #[expect( + clippy::cast_possible_truncation, + reason = "truncation is not expected to happen" + )] + blob_files.push(LinkedFile { + blob_file_id, + bytes, + len: len as usize, + on_disk_bytes, + }); + } + + Some(blob_files) + } else { + None + }) + } + + /// Gets the global table ID. + #[must_use] + fn global_id(&self) -> GlobalTableId { + (self.tree_id, self.id()).into() + } + + #[must_use] + pub fn filter_size(&self) -> u32 { + self.regions.filter.map(|x| x.size()).unwrap_or_default() + } + + #[must_use] + pub fn pinned_filter_size(&self) -> usize { + self.pinned_filter_block + .as_ref() + .map(FilterBlock::size) + .unwrap_or_default() + } + + #[must_use] + pub fn pinned_block_index_size(&self) -> usize { + match &*self.block_index { + BlockIndexImpl::Full(full_block_index) => full_block_index.inner().inner.size(), + BlockIndexImpl::VolatileFull(_) => 0, + BlockIndexImpl::TwoLevel(two_level_block_index) => { + two_level_block_index.top_level_index.inner.size() + } + } + } + + /// Gets the table ID. + /// + /// The table ID is unique for this tree, but not + /// across multiple trees, use [`Table::global_id`] for that. + #[must_use] + pub fn id(&self) -> TableId { + self.metadata.id + } + + fn load_block( + &self, + handle: &BlockHandle, + block_type: BlockType, + compression: CompressionType, + ) -> crate::Result { + load_block( + self.global_id(), + &self.path, + &self.file_accessor, + &self.cache, + handle, + block_type, + compression, + #[cfg(feature = "metrics")] + &self.metrics, + ) + } + + fn load_data_block(&self, handle: &BlockHandle) -> crate::Result { + self.load_block( + handle, + BlockType::Data, + self.metadata.data_block_compression, + ) + .map(DataBlock::new) + } + + /// Returns the (possibly compressed) file size. + pub(crate) fn file_size(&self) -> u64 { + self.metadata.file_size + } + + pub fn get( + &self, + key: &[u8], + seqno: SeqNo, + key_hash: u64, + ) -> crate::Result> { + self.get_with(key, seqno, key_hash, Self::point_read) + } + + pub(crate) fn get_value( + &self, + key: &[u8], + seqno: SeqNo, + key_hash: u64, + ) -> crate::Result> { + self.get_with(key, seqno, key_hash, Self::point_read_value) + } + + fn get_with( + &self, + key: &[u8], + seqno: SeqNo, + key_hash: u64, + point_read: impl FnOnce(&Self, &[u8], SeqNo) -> crate::Result>, + ) -> crate::Result> { + #[cfg(feature = "metrics")] + use std::sync::atomic::Ordering::Relaxed; + + // Translate seqno to "our" seqno + let seqno = seqno.saturating_sub(self.global_seqno()); + + if self.metadata.seqnos.0 >= seqno { + return Ok(None); + } + + let filter_block = if let Some(block) = &self.pinned_filter_block { + Some(Cow::Borrowed(block)) + } else if let Some(filter_idx) = &self.pinned_filter_index { + let mut iter = filter_idx.iter(); + iter.seek(key, seqno); + + if let Some(filter_block_handle) = iter.next() { + let filter_block_handle = filter_block_handle.materialize(filter_idx.as_slice()); + + let block = self.load_block( + &filter_block_handle.into_inner(), + BlockType::Filter, + CompressionType::None, // NOTE: We never write a filter block with compression + )?; + let block = FilterBlock::new(block); + + Some(Cow::Owned(block)) + } else { + None + } + } else if let Some(_filter_tli_handle) = &self.regions.filter_tli { + unimplemented!("unpinned filter TLI not supported"); + } else if let Some(filter_block_handle) = &self.regions.filter { + let block = self.load_block( + filter_block_handle, + BlockType::Filter, + CompressionType::None, // NOTE: We never write a filter block with compression + )?; + let block = FilterBlock::new(block); + + Some(Cow::Owned(block)) + } else { + None + }; + + if let Some(filter_block) = &filter_block { + if !filter_block.maybe_contains_hash(key_hash)? { + #[cfg(feature = "metrics")] + { + self.metrics.filter_queries.fetch_add(1, Relaxed); + self.metrics.io_skipped_by_filter.fetch_add(1, Relaxed); + } + + return Ok(None); + } + } + + let item = point_read(self, key, seqno); + + #[cfg(not(feature = "metrics"))] + { + item + } + + #[cfg(feature = "metrics")] + { + // NOTE: Only increment the filter queries when the filter reported a miss + // and we actually waste an I/O for a non-existing item. + // Otherwise, the filter efficiency decreases whenever an item is hit. + // https://github.com/fjall-rs/lsm-tree/issues/246 + item.inspect(|maybe_kv| { + if maybe_kv.is_none() && filter_block.is_some() { + self.metrics.filter_queries.fetch_add(1, Relaxed); + } + }) + } + } + + fn point_read(&self, key: &[u8], seqno: SeqNo) -> crate::Result> { + self.point_read_with(key, seqno, |block, key, seqno| { + block.point_read(key, seqno).map(|mut item| { + item.key.seqno += self.global_seqno(); + item + }) + }) + } + + fn point_read_value(&self, key: &[u8], seqno: SeqNo) -> crate::Result> { + self.point_read_with(key, seqno, DataBlock::point_read_value) + } + + fn point_read_with( + &self, + key: &[u8], + seqno: SeqNo, + point_read: impl Fn(&DataBlock, &[u8], SeqNo) -> Option, + ) -> crate::Result> { + let Some(iter) = self.block_index.forward_reader(key, seqno) else { + return Ok(None); + }; + + for block_handle in iter { + let block_handle = block_handle?; + let block = self.load_data_block(block_handle.as_ref())?; + + if let Some(item) = point_read(&block, key, seqno) { + return Ok(Some(item)); + } + + if block_handle.end_key() > &key { + return Ok(None); + } + } + + Ok(None) + } + + /// Creates a scanner over the `Table`. + /// + /// The scanner is ĺogically the same as a normal iter(), + /// however it uses its own file descriptor, does not look into the block cache + /// and uses buffered I/O. + /// + /// Used for compactions and thus not available to a user. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn scan(&self) -> crate::Result { + #[expect( + clippy::expect_used, + reason = "there shouldn't be 4 billion data blocks in a single table" + )] + let block_count = self + .metadata + .data_block_count + .try_into() + .expect("data block count should fit"); + + Scanner::new( + &self.path, + block_count, + self.metadata.data_block_compression, + self.global_seqno(), + ) + } + + /// Creates an iterator over the `Table`. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[must_use] + #[doc(hidden)] + pub fn iter(&self) -> impl DoubleEndedIterator> + use<> { + self.range(..) + } + + /// Creates a ranged iterator over the `Table`. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[must_use] + #[doc(hidden)] + pub fn range + Send>( + &self, + range: R, + ) -> impl DoubleEndedIterator> + Send + use { + let index_iter = self.block_index.iter(); + + let mut iter = Iter::new( + self.global_id(), + self.global_seqno(), + self.path.clone(), + index_iter, + self.file_accessor.clone(), + self.cache.clone(), + self.metadata.data_block_compression, + #[cfg(feature = "metrics")] + self.metrics.clone(), + ); + + match range.start_bound() { + Bound::Included(key) => iter.set_lower_bound(iter::Bound::Included(key.clone())), + Bound::Excluded(key) => iter.set_lower_bound(iter::Bound::Excluded(key.clone())), + Bound::Unbounded => {} + } + + match range.end_bound() { + Bound::Included(key) => iter.set_upper_bound(iter::Bound::Included(key.clone())), + Bound::Excluded(key) => iter.set_upper_bound(iter::Bound::Excluded(key.clone())), + Bound::Unbounded => {} + } + + iter + } + + fn read_tli( + regions: &ParsedRegions, + file: &File, + compression: CompressionType, + ) -> crate::Result { + log::trace!("Reading TLI block, with tli_ptr={:?}", regions.tli); + + let block = Block::from_file(file, regions.tli, compression)?; + + if block.header.block_type != BlockType::Index { + return Err(crate::Error::InvalidTag(( + "BlockType", + block.header.block_type.into(), + ))); + } + + Ok(IndexBlock::new(block)) + } + + /// Tries to recover a table from a file. + #[warn( + clippy::too_many_arguments, + clippy::too_many_lines, + reason = "TODO: refactor" + )] + pub fn recover( + file_path: PathBuf, + checksum: Checksum, + global_seqno: SeqNo, + tree_id: TreeId, + cache: Arc, + descriptor_table: Option>, + pin_filter: bool, + pin_index: bool, + #[cfg(feature = "metrics")] metrics: Arc, + ) -> crate::Result { + use meta::ParsedMeta; + use regions::ParsedRegions; + use std::sync::atomic::AtomicBool; + + log::debug!("Recovering table from file {}", file_path.display()); + let mut file = std::fs::File::open(&file_path)?; + let file_path = Arc::new(file_path); + + #[cfg(feature = "metrics")] + metrics + .table_file_opened_uncached + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + let trailer = sfa::Reader::from_reader(&mut file)?; + let regions = ParsedRegions::parse_from_toc(trailer.toc())?; + + log::trace!("Reading meta block, with meta_ptr={:?}", regions.metadata); + let metadata = ParsedMeta::load_with_handle(&file, ®ions.metadata)?; + + let file = Arc::new(file); + + let file_accessor = if let Some(dt) = descriptor_table { + FileAccessor::DescriptorTable(dt) + } else { + FileAccessor::File(file.clone()) + }; + + let block_index = if regions.index.is_some() { + log::trace!( + "Creating partitioned block index, with tli_ptr={:?}", + regions.tli, + ); + + let block = Self::read_tli(®ions, &file, metadata.index_block_compression)?; + + BlockIndexImpl::TwoLevel(TwoLevelBlockIndex { + top_level_index: block, + cache: cache.clone(), + compression: metadata.index_block_compression, + path: Arc::clone(&file_path), + file_accessor: file_accessor.clone(), + table_id: (tree_id, metadata.id).into(), + + #[cfg(feature = "metrics")] + metrics: metrics.clone(), + }) + } else if pin_index { + log::trace!( + "Creating pinned, full block index, with tli_ptr={:?}", + regions.tli, + ); + + let block = Self::read_tli(®ions, &file, metadata.index_block_compression)?; + BlockIndexImpl::Full(FullBlockIndex::new(block)) + } else { + log::trace!("Creating volatile, full block index"); + + BlockIndexImpl::VolatileFull(VolatileBlockIndex { + cache: cache.clone(), + compression: metadata.index_block_compression, + file_accessor: file_accessor.clone(), + handle: regions.tli, + path: Arc::clone(&file_path), + table_id: (tree_id, metadata.id).into(), + + #[cfg(feature = "metrics")] + metrics: metrics.clone(), + }) + }; + + let pinned_filter_index = if let Some(filter_tli_handle) = regions.filter_tli { + let block = + Block::from_file(&file, filter_tli_handle, metadata.index_block_compression)?; + Some(IndexBlock::new(block)) + } else { + None + }; + + // TODO: FilterBlock newtype + let pinned_filter_block = if pinned_filter_index.is_none() && pin_filter { + regions + .filter + .map(|filter_handle| { + log::debug!( + "Loading and pinning filter block, with filter_ptr={filter_handle:?}" + ); + + let block = Block::from_file( + &file, + filter_handle, + crate::CompressionType::None, // NOTE: We never write a filter block with compression + ) + .and_then(|block| { + if block.header.block_type == BlockType::Filter { + Ok(block) + } else { + Err(crate::Error::InvalidTag(( + "BlockType", + block.header.block_type.into(), + ))) + } + })?; + + Ok::<_, crate::Error>(FilterBlock::new(block)) + }) + .transpose()? + } else { + None + }; + + log::debug!( + "Recovered table #{} from {}", + metadata.id, + file_path.display(), + ); + + Ok(Self(Arc::new(Inner { + path: file_path, + tree_id, + + metadata, + regions, + + cache, + + file_accessor, + + block_index: Arc::new(block_index), + + pinned_filter_index, + + pinned_filter_block, + + is_deleted: AtomicBool::default(), + + checksum, + global_seqno, + + #[cfg(feature = "metrics")] + metrics, + + cached_blob_bytes: std::sync::OnceLock::new(), + }))) + } + + #[must_use] + pub fn checksum(&self) -> Checksum { + self.0.checksum + } + + pub(crate) fn mark_as_deleted(&self) { + self.0 + .is_deleted + .store(true, std::sync::atomic::Ordering::Release); + } + + /// Checks if a key range is (partially or fully) contained in this table. + pub(crate) fn check_key_range_overlap(&self, bounds: &(Bound<&[u8]>, Bound<&[u8]>)) -> bool { + self.metadata.key_range.overlaps_with_bounds(bounds) + } + + /// Returns the highest sequence number in the table. + #[must_use] + pub fn get_highest_seqno(&self) -> SeqNo { + self.metadata.seqnos.1 + self.global_seqno() + } + + /// Returns the number of tombstone markers in the `Table`. + #[must_use] + #[doc(hidden)] + pub fn tombstone_count(&self) -> u64 { + self.metadata.tombstone_count + } + + /// Returns the number of weak (single delete) tombstones in the `Table`. + #[must_use] + #[doc(hidden)] + pub fn weak_tombstone_count(&self) -> u64 { + self.metadata.weak_tombstone_count + } + + /// Returns the number of value entries reclaimable once weak tombstones can be GC'd. + #[must_use] + #[doc(hidden)] + pub fn weak_tombstone_reclaimable(&self) -> u64 { + self.metadata.weak_tombstone_reclaimable + } + + /// Returns the ratio of tombstone markers in the `Table`. + #[must_use] + #[doc(hidden)] + pub fn tombstone_ratio(&self) -> f32 { + todo!() + + // self.metadata.tombstone_count as f32 / self.metadata.key_count as f32 + } +} diff --git a/crates/lsm-tree/src/table/multi_writer.rs b/crates/lsm-tree/src/table/multi_writer.rs new file mode 100644 index 000000000..1464028e7 --- /dev/null +++ b/crates/lsm-tree/src/table/multi_writer.rs @@ -0,0 +1,350 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{filter::BloomConstructionPolicy, writer::Writer}; +use crate::{ + Checksum, CompressionType, HashMap, SequenceNumberCounter, TableId, UserKey, + blob_tree::handle::BlobIndirection, table::writer::LinkedFile, value::InternalValue, + vlog::BlobFileId, +}; +use std::path::PathBuf; + +/// Like `Writer` but will rotate to a new table, once a table grows larger than `target_size` +/// +/// This results in a sorted "run" of tables +pub struct MultiWriter { + pub(crate) base_path: PathBuf, + + data_block_hash_ratio: f32, + + data_block_size: u32, + + data_block_restart_interval: u8, + index_block_restart_interval: u8, + + use_partitioned_index: bool, + use_partitioned_filter: bool, + + /// Target size of tables in bytes + /// + /// If a table reaches the target size, a new one is started, + /// resulting in a sorted "run" of tables + pub target_size: u64, + + results: Vec<(TableId, Checksum)>, + + table_id_generator: SequenceNumberCounter, + + pub writer: Writer, + + pub data_block_compression: CompressionType, + pub index_block_compression: CompressionType, + + bloom_policy: BloomConstructionPolicy, + + current_key: Option, + has_items: bool, + + linked_blobs: HashMap, + + /// Level the tables are written to + initial_level: u8, +} + +impl MultiWriter { + /// Sets up a new `MultiWriter` at the given tables folder + pub fn new( + base_path: PathBuf, + table_id_generator: SequenceNumberCounter, + target_size: u64, + initial_level: u8, + ) -> crate::Result { + let current_table_id = table_id_generator.next(); + + let path = base_path.join(current_table_id.to_string()); + let writer = Writer::new(path, current_table_id, initial_level)?; + + Ok(Self { + initial_level, + + base_path, + + data_block_hash_ratio: 0.0, + + data_block_size: 4_096, + + data_block_restart_interval: 16, + index_block_restart_interval: 1, + + target_size, + results: Vec::new(), + table_id_generator, + writer, + + data_block_compression: CompressionType::None, + index_block_compression: CompressionType::None, + + use_partitioned_index: false, + use_partitioned_filter: false, + + bloom_policy: BloomConstructionPolicy::default(), + + current_key: None, + has_items: false, + + linked_blobs: HashMap::default(), + }) + } + + pub fn register_blob(&mut self, indirection: BlobIndirection) { + self.linked_blobs + .entry(indirection.vhandle.blob_file_id) + .and_modify(|entry| { + entry.bytes += u64::from(indirection.size); + entry.on_disk_bytes += u64::from(indirection.vhandle.on_disk_size); + entry.len += 1; + }) + .or_insert_with(|| LinkedFile { + blob_file_id: indirection.vhandle.blob_file_id, + bytes: u64::from(indirection.size), + on_disk_bytes: u64::from(indirection.vhandle.on_disk_size), + len: 1, + }); + } + + #[must_use] + pub fn use_partitioned_index(mut self) -> Self { + self.use_partitioned_index = true; + self.writer = self.writer.use_partitioned_index(); + self + } + + #[must_use] + pub fn use_partitioned_filter(mut self) -> Self { + self.use_partitioned_filter = true; + self.writer = self.writer.use_partitioned_filter(); + self + } + + #[must_use] + pub fn use_data_block_restart_interval(mut self, interval: u8) -> Self { + self.data_block_restart_interval = interval; + self.writer = self.writer.use_data_block_restart_interval(interval); + self + } + + #[must_use] + pub fn use_index_block_restart_interval(mut self, interval: u8) -> Self { + self.index_block_restart_interval = interval; + self.writer = self.writer.use_index_block_restart_interval(interval); + self + } + + #[must_use] + pub fn use_data_block_hash_ratio(mut self, ratio: f32) -> Self { + self.data_block_hash_ratio = ratio; + self.writer = self.writer.use_data_block_hash_ratio(ratio); + self + } + + #[must_use] + pub(crate) fn use_data_block_size(mut self, size: u32) -> Self { + assert!( + size <= 4 * 1_024 * 1_024, + "data block size must be <= 4 MiB", + ); + self.data_block_size = size; + self.writer = self.writer.use_data_block_size(size); + self + } + + #[must_use] + pub fn use_data_block_compression(mut self, compression: CompressionType) -> Self { + self.data_block_compression = compression; + self.writer = self.writer.use_data_block_compression(compression); + self + } + + #[must_use] + pub fn use_index_block_compression(mut self, compression: CompressionType) -> Self { + self.index_block_compression = compression; + self.writer = self.writer.use_index_block_compression(compression); + self + } + + #[must_use] + pub fn use_bloom_policy(mut self, bloom_policy: BloomConstructionPolicy) -> Self { + self.bloom_policy = bloom_policy; + self.writer = self.writer.use_bloom_policy(bloom_policy); + self + } + + /// Flushes the current writer, stores its metadata, and sets up a new writer for the next table + fn rotate(&mut self) -> crate::Result<()> { + log::debug!("Rotating table writer"); + + let new_table_id = self.table_id_generator.next(); + let path = self.base_path.join(new_table_id.to_string()); + + let mut new_writer = Writer::new(path, new_table_id, self.initial_level)? + .use_data_block_compression(self.data_block_compression) + .use_index_block_compression(self.index_block_compression) + .use_data_block_size(self.data_block_size) + .use_data_block_restart_interval(self.data_block_restart_interval) + .use_index_block_restart_interval(self.index_block_restart_interval) + .use_bloom_policy(self.bloom_policy) + .use_data_block_hash_ratio(self.data_block_hash_ratio); + + if self.use_partitioned_index { + new_writer = new_writer.use_partitioned_index(); + } + if self.use_partitioned_filter { + new_writer = new_writer.use_partitioned_filter(); + } + + let mut old_writer = std::mem::replace(&mut self.writer, new_writer); + + for linked in self.linked_blobs.values() { + old_writer.link_blob_file( + linked.blob_file_id, + linked.len, + linked.bytes, + linked.on_disk_bytes, + ); + } + self.linked_blobs.clear(); + + if let Some((table_id, checksum)) = old_writer.finish()? { + self.results.push((table_id, checksum)); + } + + Ok(()) + } + + /// Writes an item + pub fn write(&mut self, item: InternalValue) -> crate::Result<()> { + let is_next_key = self.current_key.as_ref() < Some(&item.key.user_key); + + if is_next_key { + self.current_key = Some(item.key.user_key.clone()); + + if *self.writer.meta.file_pos >= self.target_size { + self.rotate()?; + } + } + + self.writer.write(item)?; + self.has_items = true; + + Ok(()) + } + + pub(crate) fn write_distinct(&mut self, item: InternalValue) -> crate::Result<()> { + if *self.writer.meta.file_pos >= self.target_size { + self.rotate()?; + } + + self.writer.write_distinct(item)?; + self.has_items = true; + Ok(()) + } + + pub(crate) fn is_empty(&self) -> bool { + !self.has_items + } + + /// Finishes the last table, making sure all data is written durably + /// + /// Returns the metadata of created tables + pub fn finish(mut self) -> crate::Result> { + for linked in self.linked_blobs.values() { + self.writer.link_blob_file( + linked.blob_file_id, + linked.len, + linked.bytes, + linked.on_disk_bytes, + ); + } + + if let Some((table_id, checksum)) = self.writer.finish()? { + self.results.push((table_id, checksum)); + } + + Ok(self.results) + } +} + +#[cfg(test)] +mod tests { + use crate::{AbstractTree, Config, SeqNo, SequenceNumberCounter, config::CompressionPolicy}; + use test_log::test; + + // NOTE: Tests that versions of the same key stay + // in the same table even if it needs to be rotated + // + // This avoids tables' key ranges overlapping + // + // http://github.com/fjall-rs/lsm-tree/commit/f46b6fe26a1e90113dc2dbb0342db160a295e616 + #[test] + fn table_multi_writer_same_key_norotate() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_compression_policy(CompressionPolicy::all(crate::CompressionType::None)) + .index_block_compression_policy(CompressionPolicy::all(crate::CompressionType::None)) + .open()?; + + tree.insert("a", "a1".repeat(4_000), 0); + tree.insert("a", "a2".repeat(4_000), 1); + tree.insert("a", "a3".repeat(4_000), 2); + tree.insert("a", "a4".repeat(4_000), 3); + tree.insert("a", "a5".repeat(4_000), 4); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + + tree.major_compact(1_024, 0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + + Ok(()) + } + + // NOTE: Follow-up fix for non-disjoint output + // + // https://github.com/fjall-rs/lsm-tree/commit/1609a57c2314420b858d826790ecd1442aa76720 + #[test] + fn table_multi_writer_same_key_norotate_2() -> crate::Result<()> { + let folder = tempfile::tempdir()?; + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_compression_policy(CompressionPolicy::all(crate::CompressionType::None)) + .index_block_compression_policy(CompressionPolicy::all(crate::CompressionType::None)) + .open()?; + + tree.insert("a", "a1".repeat(4_000), 0); + tree.insert("a", "a1".repeat(4_000), 1); + tree.insert("a", "a1".repeat(4_000), 2); + tree.insert("b", "a1".repeat(4_000), 0); + tree.insert("c", "a1".repeat(4_000), 0); + tree.insert("c", "a1".repeat(4_000), 1); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(3, tree.len(SeqNo::MAX, None)?); + + tree.major_compact(1_024, 0)?; + assert_eq!(3, tree.table_count()); + assert_eq!(3, tree.len(SeqNo::MAX, None)?); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/table/regions.rs b/crates/lsm-tree/src/table/regions.rs new file mode 100644 index 000000000..5644d2f2b --- /dev/null +++ b/crates/lsm-tree/src/table/regions.rs @@ -0,0 +1,77 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{BlockHandle, BlockOffset}; +use sfa::TocEntry; + +/// Converts a [`sfa::TocEntry`] to our [`BlockHandle`] struct. +fn toc_entry_to_handle(entry: &TocEntry) -> BlockHandle { + #[expect( + clippy::expect_used, + reason = "Function is only used for regions that do not exceed >= 4 GiB" + )] + BlockHandle::new( + BlockOffset(entry.pos()), + entry + .len() + .try_into() + .expect("region should not exceed 4 GiB"), + ) +} + +/// The regions block stores offsets to the different table file "regions" +/// +/// ---------------- +/// | data | <- implicitly start at 0 +/// |--------------| +/// | tli | +/// |--------------| +/// | index | <- may not exist (if full block index is used, TLI will be dense) +/// |--------------| +/// | filter | <- may not exist +/// |--------------| +/// | ... | +/// |--------------| +/// | linked blobs | <- may not exist +/// |--------------| +/// | meta | +/// |--------------| +/// | toc | +/// |--------------| +/// | trailer | <- fixed size +/// |--------------| +#[derive(Copy, Clone, Debug, Default)] +pub struct ParsedRegions { + pub tli: BlockHandle, + pub index: Option, + pub filter_tli: Option, + pub filter: Option, + pub linked_blob_files: Option, + pub metadata: BlockHandle, +} + +impl ParsedRegions { + pub fn parse_from_toc(toc: &sfa::Toc) -> crate::Result { + Ok(Self { + filter_tli: toc.section(b"filter_tli").map(toc_entry_to_handle), + tli: toc + .section(b"tli") + .map(toc_entry_to_handle) + .ok_or_else(|| { + log::error!("TLI should exist"); + crate::Error::Unrecoverable + })?, + index: toc.section(b"index").map(toc_entry_to_handle), + filter: toc.section(b"filter").map(toc_entry_to_handle), + linked_blob_files: toc.section(b"linked_blob_files").map(toc_entry_to_handle), + metadata: toc + .section(b"meta") + .map(toc_entry_to_handle) + .ok_or_else(|| { + log::error!("Metadata should exist"); + crate::Error::Unrecoverable + })?, + }) + } +} diff --git a/crates/lsm-tree/src/table/scanner.rs b/crates/lsm-tree/src/table/scanner.rs new file mode 100644 index 000000000..b1e1f4c0e --- /dev/null +++ b/crates/lsm-tree/src/table/scanner.rs @@ -0,0 +1,93 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Block, DataBlock}; +use crate::{ + CompressionType, InternalValue, SeqNo, + table::{block::BlockType, iter::OwnedDataBlockIter}, +}; +use std::{fs::File, io::BufReader, path::Path}; + +/// Table reader that is optimized for consuming an entire table +pub struct Scanner { + reader: BufReader, + iter: OwnedDataBlockIter, + + compression: CompressionType, + block_count: usize, + read_count: usize, + + global_seqno: SeqNo, +} + +impl Scanner { + pub fn new( + path: &Path, + block_count: usize, + compression: CompressionType, + global_seqno: SeqNo, + ) -> crate::Result { + // TODO: a larger buffer size may be better for HDD, maybe make this configurable + // TODO: benchmarks were inconclusive on SSD, not much difference between 4KB - 2MB + let mut reader = BufReader::with_capacity(8 * 4_096, File::open(path)?); + + let block = Self::fetch_next_block(&mut reader, compression)?; + let iter = OwnedDataBlockIter::new(block, DataBlock::iter); + + Ok(Self { + reader, + iter, + + compression, + block_count, + read_count: 1, + + global_seqno, + }) + } + + fn fetch_next_block( + reader: &mut BufReader, + compression: CompressionType, + ) -> crate::Result { + let block = Block::from_reader(reader, compression); + + match block { + Ok(block) => { + if block.header.block_type != BlockType::Data { + return Err(crate::Error::InvalidTag(( + "BlockType", + block.header.block_type.into(), + ))); + } + + Ok(DataBlock::new(block)) + } + Err(e) => Err(e), + } + } +} + +impl Iterator for Scanner { + type Item = crate::Result; + + fn next(&mut self) -> Option { + loop { + if let Some(mut item) = self.iter.next() { + item.key.seqno += self.global_seqno; + return Some(Ok(item)); + } + + if self.read_count >= self.block_count { + return None; + } + + // Init new block + let block = fail_iter!(Self::fetch_next_block(&mut self.reader, self.compression)); + self.iter = OwnedDataBlockIter::new(block, DataBlock::iter); + + self.read_count += 1; + } + } +} diff --git a/crates/lsm-tree/src/table/tests.rs b/crates/lsm-tree/src/table/tests.rs new file mode 100644 index 000000000..622564936 --- /dev/null +++ b/crates/lsm-tree/src/table/tests.rs @@ -0,0 +1,1474 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::*; +use crate::{ + config::BloomConstructionPolicy, table::filter::standard_bloom::Builder as BloomBuilder, +}; +use tempfile::tempdir; +use test_log::test; + +#[expect( + clippy::too_many_lines, + clippy::cognitive_complexity, + clippy::cast_possible_truncation, + clippy::unwrap_used +)] +fn test_with_table( + items: &[InternalValue], + f: impl Fn(Table) -> crate::Result<()>, + rotate_every: Option, + config_writer: Option Writer>, +) -> crate::Result<()> { + let dir = tempdir()?; + let file = dir.path().join("table"); + + { + let mut writer = Writer::new(file.clone(), 0, 0)?; + + if let Some(f) = &config_writer { + writer = f(writer); + } + + for (idx, item) in items.iter().enumerate() { + if let Some(rotate) = rotate_every { + if idx % rotate == 0 { + writer.spill_block()?; + } + } + writer.write(item.clone())?; + } + let (_, checksum) = writer.finish()?.unwrap(); + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + false, + false, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_none(), "should use full index"); + assert_eq!(0, table.pinned_block_index_size(), "should not pin index"); + assert_eq!(0, table.pinned_filter_size(), "should not pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + true, + false, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_none(), "should use full index"); + assert_eq!(0, table.pinned_block_index_size(), "should not pin index"); + // assert!(table.pinned_filter_size() > 0, "should pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + false, + true, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_none(), "should use full index"); + assert!(table.pinned_block_index_size() > 0, "should pin index"); + assert_eq!(0, table.pinned_filter_size(), "should not pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + true, + true, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_none(), "should use full index"); + assert!(table.pinned_block_index_size() > 0, "should pin index"); + // assert!(table.pinned_filter_size() > 0, "should pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + None, + true, + true, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_none(), "should use full index"); + assert!(table.pinned_block_index_size() > 0, "should pin index"); + // assert!(table.pinned_filter_size() > 0, "should pin filter"); + assert!(matches!(table.file_accessor, FileAccessor::File(..))); + + f(table)?; + } + } + + std::fs::remove_file(&file)?; + + // Test with partitioned indexes + { + let mut writer = Writer::new(file.clone(), 0, 0)?.use_partitioned_index(); + + if let Some(f) = config_writer { + writer = f(writer); + } + + for (idx, item) in items.iter().enumerate() { + if let Some(rotate) = rotate_every { + if idx % rotate == 0 { + writer.spill_block()?; + } + } + writer.write(item.clone())?; + } + let (_, checksum) = writer.finish()?.unwrap(); + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + false, + false, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_some(), "should use two-level index",); + assert_eq!(0, table.pinned_filter_size(), "should not pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + true, + false, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_some(), "should use two-level index",); + // assert!(table.pinned_filter_size() > 0, "should pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + false, + true, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_some(), "should use two-level index",); + assert!(table.pinned_block_index_size() > 0, "should pin index"); + // assert_eq!(0, table.pinned_filter_size(), "should not pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file.clone(), + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + Some(Arc::new(DescriptorTable::new(10))), + true, + true, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_some(), "should use two-level index",); + assert!(table.pinned_block_index_size() > 0, "should pin index"); + // assert!(table.pinned_filter_size() > 0, "should pin filter"); + assert!(matches!( + table.file_accessor, + FileAccessor::DescriptorTable(..) + )); + + f(table)?; + } + + { + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let table = Table::recover( + file, + checksum, + 0, + 0, + Arc::new(Cache::with_capacity_bytes(1_000_000)), + None, + true, + true, + #[cfg(feature = "metrics")] + metrics, + )?; + + assert_eq!(0, table.id()); + assert_eq!(items.len(), table.metadata.item_count as usize); + assert!(table.regions.index.is_some(), "should use two-level index",); + assert!(table.pinned_block_index_size() > 0, "should pin index"); + // assert!(table.pinned_filter_size() > 0, "should pin filter"); + assert!(matches!(table.file_accessor, FileAccessor::File(..))); + + f(table)?; + } + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_point_read() -> crate::Result<()> { + let items = [crate::InternalValue::from_components( + b"abc", + b"asdasdasd", + 3, + crate::ValueType::Value, + )]; + + test_with_table( + &items, + |table| { + assert_eq!( + b"abc", + &*table + .get(b"abc", SeqNo::MAX, BloomBuilder::get_hash(b"abc"))? + .unwrap() + .key + .user_key, + ); + assert_eq!( + None, + table.get(b"def", SeqNo::MAX, BloomBuilder::get_hash(b"def"))?, + ); + assert_eq!( + None, + table.get(b"____", SeqNo::MAX, BloomBuilder::get_hash(b"____"))?, + ); + + assert_eq!( + table.metadata.key_range, + crate::KeyRange::new((b"abc".into(), b"abc".into())), + ); + + Ok(()) + }, + None, + Some(|x| x), + ) +} + +#[test] +fn table_range_exclusive_bounds() -> crate::Result<()> { + use std::ops::Bound::{Excluded, Included}; + + let items = [ + crate::InternalValue::from_components(b"a", b"v", 0, crate::ValueType::Value), + crate::InternalValue::from_components(b"b", b"v", 0, crate::ValueType::Value), + crate::InternalValue::from_components(b"c", b"v", 0, crate::ValueType::Value), + crate::InternalValue::from_components(b"d", b"v", 0, crate::ValueType::Value), + crate::InternalValue::from_components(b"e", b"v", 0, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + let res = table + .range((Excluded(UserKey::from("b")), Included(UserKey::from("d")))) + .flatten() + .collect::>(); + assert_eq!( + items.iter().skip(2).take(2).cloned().collect::>(), + &*res, + ); + + let res = table + .range((Excluded(UserKey::from("b")), Included(UserKey::from("d")))) + .rev() + .flatten() + .collect::>(); + assert_eq!( + items + .iter() + .skip(2) + .take(2) + .rev() + .cloned() + .collect::>(), + &*res, + ); + + let res = table + .range((Excluded(UserKey::from("b")), Excluded(UserKey::from("d")))) + .flatten() + .collect::>(); + assert_eq!( + items.iter().skip(2).take(1).cloned().collect::>(), + &*res, + ); + + let res = table + .range((Excluded(UserKey::from("b")), Excluded(UserKey::from("d")))) + .rev() + .flatten() + .collect::>(); + assert_eq!( + items + .iter() + .skip(2) + .take(1) + .rev() + .cloned() + .collect::>(), + &*res, + ); + + Ok(()) + }, + None, + Some(|x: Writer| x.use_data_block_size(1)), + ) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_point_read_mvcc_block_boundary() -> crate::Result<()> { + let items = [ + crate::InternalValue::from_components(b"a", b"5", 5, crate::ValueType::Value), + crate::InternalValue::from_components(b"a", b"4", 4, crate::ValueType::Value), + crate::InternalValue::from_components(b"a", b"3", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"a", b"2", 2, crate::ValueType::Value), + crate::InternalValue::from_components(b"a", b"1", 1, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!(2, table.metadata.data_block_count); + + let key_hash = BloomBuilder::get_hash(b"a"); + + assert_eq!( + b"5", + &*table.get(b"a", SeqNo::MAX, key_hash)?.unwrap().value + ); + assert_eq!(b"4", &*table.get(b"a", 5, key_hash)?.unwrap().value); + assert_eq!(b"3", &*table.get(b"a", 4, key_hash)?.unwrap().value); + assert_eq!(b"2", &*table.get(b"a", 3, key_hash)?.unwrap().value); + assert_eq!(b"1", &*table.get(b"a", 2, key_hash)?.unwrap().value); + + Ok(()) + }, + Some(3), + Some(|x| x), + ) +} + +#[test] +fn table_scan() -> crate::Result<()> { + let items = [ + crate::InternalValue::from_components(b"abc", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"def", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"xyz", b"asdasdasd", 3, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!(items, &*table.scan()?.flatten().collect::>()); + + assert_eq!( + table.metadata.key_range, + crate::KeyRange::new((b"abc".into(), b"xyz".into())), + ); + + Ok(()) + }, + None, + Some(|x| x), + ) +} + +#[test] +fn table_iter_simple() -> crate::Result<()> { + let items = [ + crate::InternalValue::from_components(b"abc", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"def", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"xyz", b"asdasdasd", 3, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!(items, &*table.iter().flatten().collect::>()); + assert_eq!( + items.iter().rev().cloned().collect::>(), + &*table.iter().rev().flatten().collect::>(), + ); + + Ok(()) + }, + None, + Some(|x| x), + ) +} + +#[test] +fn table_range_simple() -> crate::Result<()> { + let items = [ + crate::InternalValue::from_components(b"abc", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"def", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"xyz", b"asdasdasd", 3, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!( + items.iter().skip(1).cloned().collect::>(), + &*table + .range(UserKey::from("b")..) + .flatten() + .collect::>() + ); + + assert_eq!( + items.iter().skip(1).rev().cloned().collect::>(), + &*table + .range(UserKey::from("b")..) + .rev() + .flatten() + .collect::>(), + ); + + Ok(()) + }, + None, + Some(|x| x), + ) +} + +#[test] +fn table_range_ping_pong() -> crate::Result<()> { + let items = (0u64..10) + .map(|i| InternalValue::from_components(i.to_be_bytes(), "", 0, crate::ValueType::Value)) + .collect::>(); + + test_with_table( + &items, + |table| { + let mut iter = + table.range(UserKey::from(5u64.to_be_bytes())..UserKey::from(10u64.to_be_bytes())); + + let mut count = 0; + + for x in 0.. { + if x % 2 == 0 { + let Some(_) = iter.next() else { + break; + }; + + count += 1; + } else { + let Some(_) = iter.next_back() else { + break; + }; + + count += 1; + } + } + + assert_eq!(5, count); + + Ok(()) + }, + None, + Some(|x| x), + ) +} + +#[test] +fn table_range_multiple_data_blocks() -> crate::Result<()> { + let items = [ + crate::InternalValue::from_components(b"a", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"b", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"c", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"d", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"e", b"asdasdasd", 3, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!(5, table.metadata.data_block_count); + + assert_eq!( + items.iter().skip(1).take(3).cloned().collect::>(), + &*table + .range(UserKey::from("b")..=UserKey::from("d")) + .flatten() + .collect::>() + ); + + assert_eq!( + items + .iter() + .skip(1) + .take(3) + .rev() + .cloned() + .collect::>(), + &*table + .range(UserKey::from("b")..=UserKey::from("d")) + .rev() + .flatten() + .collect::>(), + ); + + Ok(()) + }, + None, + Some(|x: Writer| x.use_data_block_size(1)), + ) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_point_read_partitioned_filter_smoke_test() -> crate::Result<()> { + let items = [ + crate::InternalValue::from_components(b"a", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"b", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"c", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"d", b"asdasdasd", 3, crate::ValueType::Value), + crate::InternalValue::from_components(b"e", b"asdasdasd", 3, crate::ValueType::Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!(1, table.metadata.data_block_count); + + for item in &items { + let key_hash = BloomBuilder::get_hash(&item.key.user_key); + + assert_eq!( + item.value, + table + .get(&item.key.user_key, SeqNo::MAX, key_hash) + .unwrap() + .unwrap() + .value, + ); + } + + Ok(()) + }, + None, + Some(|x: Writer| x.use_partitioned_filter()), + ) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_partitioned_filter() -> crate::Result<()> { + use crate::ValueType::Value; + + let items = [ + InternalValue::from_components("a", "a7", 7, Value), + InternalValue::from_components("a", "a6", 6, Value), + InternalValue::from_components("a", "a5", 5, Value), + InternalValue::from_components("a", "a4", 4, Value), + InternalValue::from_components("a", "a3", 3, Value), + InternalValue::from_components("b", "b5", 5, Value), + InternalValue::from_components("c", "c8", 8, Value), + InternalValue::from_components("d", "d10", 10, Value), + ]; + + test_with_table( + &items, + |table| { + assert!(table.regions.filter.is_some(), "filter should exist"); + assert!( + table.regions.filter_tli.is_some(), + "filter TLI should exist" + ); + + assert_eq!( + b"a7", + &*table + .get(b"a", 8, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a6", + &*table + .get(b"a", 7, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a5", + &*table + .get(b"a", 6, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a4", + &*table + .get(b"a", 5, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a3", + &*table + .get(b"a", 4, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"b5", + &*table + .get(b"b", 6, BloomBuilder::get_hash(b"b"))? + .unwrap() + .value, + ); + assert_eq!( + b"c8", + &*table + .get(b"c", 9, BloomBuilder::get_hash(b"c"))? + .unwrap() + .value, + ); + assert_eq!( + b"d10", + &*table + .get(b"d", 11, BloomBuilder::get_hash(b"d"))? + .unwrap() + .value, + ); + Ok(()) + }, + None, + Some(|x: Writer| x.use_partitioned_filter().use_meta_partition_size(3)), + ) +} + +#[test] +fn table_seqnos() -> crate::Result<()> { + use crate::ValueType::Value; + + let items = [ + InternalValue::from_components("a", nanoid::nanoid!().as_bytes(), 7, Value), + InternalValue::from_components("b", nanoid::nanoid!().as_bytes(), 5, Value), + InternalValue::from_components("c", nanoid::nanoid!().as_bytes(), 8, Value), + InternalValue::from_components("d", nanoid::nanoid!().as_bytes(), 10, Value), + ]; + + test_with_table( + &items, + |table| { + assert_eq!(5, table.metadata.seqnos.0); + assert_eq!(10, table.metadata.seqnos.1); + Ok(()) + }, + None, + Some(|x| x), + ) +} + +#[test] +fn table_zero_bpk() -> crate::Result<()> { + use crate::ValueType::Value; + + let items = [ + InternalValue::from_components("a", nanoid::nanoid!().as_bytes(), 7, Value), + InternalValue::from_components("b", nanoid::nanoid!().as_bytes(), 5, Value), + InternalValue::from_components("c", nanoid::nanoid!().as_bytes(), 8, Value), + InternalValue::from_components("d", nanoid::nanoid!().as_bytes(), 10, Value), + ]; + + test_with_table( + &items, + |table| { + assert!(table.regions.filter.is_none()); + Ok(()) + }, + None, + Some(|x: Writer| x.use_bloom_policy(BloomConstructionPolicy::BitsPerKey(0.0))), + ) +} + +#[test] +#[expect( + clippy::unreadable_literal, + clippy::unwrap_used, + clippy::indexing_slicing, + clippy::cast_possible_truncation +)] +#[cfg(not(feature = "metrics"))] +fn table_read_fuzz_1() -> crate::Result<()> { + use crate::Slice; + use crate::ValueType::{Tombstone, Value}; + + let items = [ + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 18340908174618760209, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 18054235897395861447, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([103]), + 17820711698989577060, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 17652351990810576660, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 17576667967203573449, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([30]), + 16889403751796995588, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([186]), + 15595956295177086731, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 15512796775024989213, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([188, 156, 59, 85, 13]), + 15149465603839159843, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([174, 71]), + 15102256701513339307, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([35, 148]), + 15091160407760527013, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 14675333203365509622, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([245]), + 14571905818510788533, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 14541113699969547298, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 14486387191240337417, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 14112006182482717758, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([159]), + 13992512869528291746, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 13915106262991388976, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 13597506620670366065, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 13064400463180401957, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 12969967266897711474, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 12508372658468564628, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([138]), + 11795269606598686255, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([18]), + 10730214428751858128, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([236]), + 10124645034840293700, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([216, 81]), + 9559308046784608794, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([79]), + 8607115510826103394, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 7963767336149785641, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 7882646634183551394, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 7719307175583565930, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([111]), + 7522791039398476411, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([227, 164, 129]), + 7410771579448817672, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 7003757491682295965, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 5723101273557106371, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 5581364419922287132, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([119, 29]), + 5541782075650463683, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 5136199042703471864, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 5051972816573966850, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([162]), + 5020119417385108821, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([69]), + 4325966282181409009, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 4238714774310338082, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 4200824275757201410, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([92, 145, 251, 240, 133]), + 3894954012280195585, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([14]), + 3814525464013269105, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 3766663710061910506, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([129]), + 3749655073597306832, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([231]), + 3319226033273656005, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 3274394613296787928, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 2045761581956846404, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([78]), + 1704041985603476880, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([]), + 1441130125005023946, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([164, 136]), + 1225420702887300153, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([55]), + 974698856173325051, + Value, + ), + InternalValue::from_components( + Slice::from([0]), + Slice::from([238, 237]), + 47340610649818236, + Value, + ), + InternalValue::from_components(Slice::from([0]), Slice::from([]), 0, Value), + InternalValue::from_components( + Slice::from([0, 161]), + Slice::from([]), + 17872519117933825384, + Tombstone, + ), + InternalValue::from_components( + Slice::from([0, 161]), + Slice::from([]), + 4494664966150999400, + Tombstone, + ), + InternalValue::from_components( + Slice::from([1]), + Slice::from([]), + 15373275907316083975, + Value, + ), + ]; + + let dir = tempfile::tempdir()?; + let file = dir.path().join("table_fuzz"); + + let data_block_size = 97; + + let mut writer = crate::table::Writer::new(file.clone(), 0, 0) + .unwrap() + .use_data_block_size(data_block_size); + + for item in items.iter().cloned() { + writer.write(item).unwrap(); + } + + let _trailer = writer.finish().unwrap(); + + let table = crate::Table::recover( + file, + crate::Checksum::from_raw(0), + 0, + 0, + Arc::new(crate::Cache::with_capacity_bytes(0)), + Some(Arc::new(crate::DescriptorTable::new(10))), + true, + true, + ) + .unwrap(); + + let item_count_usize = table.metadata.item_count as usize; + assert_eq!(item_count_usize, items.len()); + + assert_eq!(items.len(), item_count_usize); + let items = items.into_iter().collect::>(); + + assert_eq!(items, table.iter().collect::, _>>().unwrap()); + assert_eq!( + items.iter().rev().cloned().collect::>(), + table.iter().rev().collect::, _>>().unwrap(), + ); + + { + let lo = 0; + let hi = 54; + + let lo_key = &items[lo].key.user_key; + let hi_key = &items[hi].key.user_key; + + assert_eq!(lo_key, hi_key); + + let expected_range: Vec<_> = items[lo..=hi].to_vec(); + + let iter = table.range(lo_key..=hi_key); + + assert_eq!(expected_range, iter.collect::, _>>().unwrap()); + } + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_partitioned_index() -> crate::Result<()> { + use crate::ValueType::Value; + + let items = [ + InternalValue::from_components("a", "a7", 7, Value), + InternalValue::from_components("a", "a6", 6, Value), + InternalValue::from_components("a", "a5", 5, Value), + InternalValue::from_components("a", "a4", 4, Value), + InternalValue::from_components("a", "a3", 3, Value), + InternalValue::from_components("b", "b5", 5, Value), + InternalValue::from_components("c", "c8", 8, Value), + InternalValue::from_components("d", "d10", 10, Value), + ]; + + let dir = tempfile::tempdir()?; + let file = dir.path().join("table_fuzz"); + + let mut writer = crate::table::Writer::new(file.clone(), 0, 0) + .unwrap() + .use_partitioned_index() + .use_data_block_size(5) + .use_meta_partition_size(3); + + for item in items.iter().cloned() { + writer.write(item).unwrap(); + } + + let _trailer = writer.finish().unwrap(); + + let table = crate::Table::recover( + file, + crate::Checksum::from_raw(0), + 0, + 0, + Arc::new(crate::Cache::with_capacity_bytes(0)), + Some(Arc::new(crate::DescriptorTable::new(10))), + true, + true, + #[cfg(feature = "metrics")] + Default::default(), + ) + .unwrap(); + + assert!( + table.regions.index.is_some(), + "2nd-level index should exist", + ); + + assert!( + table.metadata.index_block_count > 1, + "should use partitioned index", + ); + + assert_eq!( + b"a7", + &*table + .get(b"a", 8, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a6", + &*table + .get(b"a", 7, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a5", + &*table + .get(b"a", 6, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a4", + &*table + .get(b"a", 5, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"a3", + &*table + .get(b"a", 4, BloomBuilder::get_hash(b"a"))? + .unwrap() + .value, + ); + assert_eq!( + b"b5", + &*table + .get(b"b", 6, BloomBuilder::get_hash(b"b"))? + .unwrap() + .value, + ); + assert_eq!( + b"c8", + &*table + .get(b"c", 9, BloomBuilder::get_hash(b"c"))? + .unwrap() + .value, + ); + assert_eq!( + b"d10", + &*table + .get(b"d", 11, BloomBuilder::get_hash(b"d"))? + .unwrap() + .value, + ); + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_global_seqno() -> crate::Result<()> { + use crate::ValueType::Value; + + let items = [ + InternalValue::from_components("a0", "a0", 0, Value), + InternalValue::from_components("a1", "a1", 1, Value), + InternalValue::from_components("b", "b", 8, Value), + ]; + + let dir = tempfile::tempdir()?; + let file = dir.path().join("table_fuzz"); + + let mut writer = crate::table::Writer::new(file.clone(), 0, 0)? + .use_partitioned_filter() + .use_data_block_size(1) + .use_meta_partition_size(1); + + for item in items { + writer.write(item)?; + } + + let _trailer = writer.finish()?; + + let table = crate::Table::recover( + file, + crate::Checksum::from_raw(0), + 7, + 0, + Arc::new(crate::Cache::with_capacity_bytes(0)), + Some(Arc::new(crate::DescriptorTable::new(10))), + true, + true, + #[cfg(feature = "metrics")] + Default::default(), + )?; + + // global seqno is 7, so a1 is = 8 -> can not be read by snapshot=8 + assert!( + table + .get(b"a1", 8, BloomBuilder::get_hash(b"a1"))? + .is_none() + ); + + assert_eq!( + b"a0", + &*table + .get(b"a0", 8, BloomBuilder::get_hash(b"a0"))? + .unwrap() + .value, + ); + + Ok(()) +} + +#[test] +#[expect(clippy::unwrap_used)] +fn table_return_global_seqno() -> crate::Result<()> { + use crate::ValueType::Value; + + const SEQNO: SeqNo = 15; + + let items = [InternalValue::from_components("abc", "abc", 0, Value)]; + + let dir = tempfile::tempdir()?; + let file = dir.path().join("table_fuzz"); + + let mut writer = crate::table::Writer::new(file.clone(), 0, 0)?; + + for item in items { + writer.write(item)?; + } + + let _trailer = writer.finish()?; + + let table = crate::Table::recover( + file, + crate::Checksum::from_raw(0), + SEQNO, + 0, + Arc::new(crate::Cache::with_capacity_bytes(0)), + Some(Arc::new(crate::DescriptorTable::new(10))), + true, + true, + #[cfg(feature = "metrics")] + Default::default(), + )?; + + assert_eq!( + InternalValue::from_components("abc", "abc", SEQNO, Value), + table + .get(b"abc", 2 * SEQNO, BloomBuilder::get_hash(b"abc"))? + .unwrap(), + ); + + Ok(()) +} diff --git a/crates/lsm-tree/src/table/util.rs b/crates/lsm-tree/src/table/util.rs new file mode 100644 index 000000000..95dd8f6a7 --- /dev/null +++ b/crates/lsm-tree/src/table/util.rs @@ -0,0 +1,224 @@ +// Copyright (c) 2025-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::{Block, BlockHandle, GlobalTableId}; +use crate::{ + Cache, CompressionType, KeyRange, Table, file_accessor::FileAccessor, table::block::BlockType, + version::run::Ranged, +}; +use std::{path::Path, sync::Arc}; + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +#[must_use] +pub fn aggregate_run_key_range(tables: &[Table]) -> KeyRange { + #[expect(clippy::expect_used, reason = "runs are never empty by definition")] + let lo = tables.first().expect("run should never be empty"); + #[expect(clippy::expect_used, reason = "runs are never empty by definition")] + let hi = tables.last().expect("run should never be empty"); + KeyRange::new((lo.key_range().min().clone(), hi.key_range().max().clone())) +} + +/// [start, end] slice indexes +#[derive(Debug)] +pub struct SliceIndexes(pub usize, pub usize); + +/// Loads a block from disk or block cache, if cached. +/// +/// Also handles file descriptor opening and caching. +#[warn(clippy::too_many_arguments)] +pub fn load_block( + table_id: GlobalTableId, + path: &Path, + file_accessor: &FileAccessor, + cache: &Cache, + handle: &BlockHandle, + block_type: BlockType, + compression: CompressionType, + #[cfg(feature = "metrics")] metrics: &Metrics, +) -> crate::Result { + #[cfg(feature = "metrics")] + use std::sync::atomic::Ordering::Relaxed; + + log::trace!("load {block_type:?} block {handle:?}"); + + if let Some(block) = cache.get_block(table_id, handle.offset()) { + #[cfg(feature = "metrics")] + match block_type { + BlockType::Filter => { + metrics.filter_block_load_cached.fetch_add(1, Relaxed); + } + BlockType::Index => { + metrics.index_block_load_cached.fetch_add(1, Relaxed); + } + BlockType::Data | BlockType::Meta => { + metrics.data_block_load_cached.fetch_add(1, Relaxed); + } + } + + return Ok(block); + } + + let (fd, fd_cache_miss) = if let Some(cached_fd) = file_accessor.access_for_table(&table_id) { + #[cfg(feature = "metrics")] + metrics.table_file_opened_cached.fetch_add(1, Relaxed); + + (cached_fd, false) + } else { + let fd = std::fs::File::open(path)?; + + #[cfg(feature = "metrics")] + metrics.table_file_opened_uncached.fetch_add(1, Relaxed); + + (Arc::new(fd), true) + }; + + let block = Block::from_file(&fd, *handle, compression)?; + + if block.header.block_type != block_type { + return Err(crate::Error::InvalidTag(( + "BlockType", + block.header.block_type.into(), + ))); + } + + #[cfg(feature = "metrics")] + match block_type { + BlockType::Filter => { + metrics.filter_block_load_io.fetch_add(1, Relaxed); + + metrics + .filter_block_io_requested + .fetch_add(handle.size().into(), Relaxed); + } + BlockType::Index => { + metrics.index_block_load_io.fetch_add(1, Relaxed); + + metrics + .index_block_io_requested + .fetch_add(handle.size().into(), Relaxed); + } + BlockType::Data | BlockType::Meta => { + metrics.data_block_load_io.fetch_add(1, Relaxed); + + metrics + .data_block_io_requested + .fetch_add(handle.size().into(), Relaxed); + } + } + + // Cache FD + if fd_cache_miss { + file_accessor.insert_for_table(table_id, fd); + } + + cache.insert_block(table_id, handle.offset(), block.clone()); + + Ok(block) +} + +#[must_use] +pub fn longest_shared_prefix_length(s1: &[u8], s2: &[u8]) -> usize { + s1.iter() + .zip(s2.iter()) + .take_while(|(c1, c2)| c1 == c2) + .count() +} + +#[must_use] +pub fn compare_prefixed_slice(prefix: &[u8], suffix: &[u8], needle: &[u8]) -> std::cmp::Ordering { + use std::cmp::Ordering::{Equal, Greater}; + + if needle.is_empty() { + let combined_len = prefix.len() + suffix.len(); + return if combined_len > 0 { Greater } else { Equal }; + } + + let max_pfx_len = prefix.len().min(needle.len()); + + { + #[expect(unsafe_code, reason = "We checked for max_pfx_len")] + let prefix = unsafe { prefix.get_unchecked(0..max_pfx_len) }; + + #[expect(unsafe_code, reason = "We checked for max_pfx_len")] + let needle = unsafe { needle.get_unchecked(0..max_pfx_len) }; + + match prefix.cmp(needle) { + Equal => {} + ordering => return ordering, + } + } + + let rest_len = prefix.len().saturating_sub(needle.len()); + if rest_len > 0 { + return Greater; + } + + #[expect( + unsafe_code, + reason = "We know that the prefix is definitely not longer than the needle so we can safely truncate" + )] + let needle = unsafe { needle.get_unchecked(max_pfx_len..) }; + suffix.cmp(needle) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn test_longest_shared_prefix_length() { + assert_eq!(3, longest_shared_prefix_length(b"abc", b"abc")); + assert_eq!(1, longest_shared_prefix_length(b"abc", b"a")); + assert_eq!(1, longest_shared_prefix_length(b"a", b"abc")); + assert_eq!(0, longest_shared_prefix_length(b"abc", b"")); + assert_eq!(0, longest_shared_prefix_length(b"", b"abc")); + assert_eq!(0, longest_shared_prefix_length(b"", b"")); + assert_eq!(0, longest_shared_prefix_length(b"", b"")); + assert_eq!(0, longest_shared_prefix_length(b"abc", b"def")); + assert_eq!(1, longest_shared_prefix_length(b"abc", b"acc")); + } + + #[test] + fn test_compare_prefixed_slice() { + use std::cmp::Ordering::{Equal, Greater, Less}; + + assert_eq!(Greater, compare_prefixed_slice(&[0, 161], &[], &[0])); + + assert_eq!(Equal, compare_prefixed_slice(b"abc", b"xyz", b"abcxyz")); + assert_eq!(Equal, compare_prefixed_slice(b"abc", b"", b"abc")); + assert_eq!(Equal, compare_prefixed_slice(b"abc", b"abc", b"abcabc")); + assert_eq!(Equal, compare_prefixed_slice(b"", b"", b"")); + assert_eq!(Less, compare_prefixed_slice(b"a", b"", b"y")); + assert_eq!(Less, compare_prefixed_slice(b"a", b"", b"yyy")); + assert_eq!(Less, compare_prefixed_slice(b"a", b"", b"yyy")); + assert_eq!(Less, compare_prefixed_slice(b"yyyy", b"a", b"yyyyb")); + assert_eq!(Less, compare_prefixed_slice(b"yyy", b"b", b"yyyyb")); + assert_eq!(Less, compare_prefixed_slice(b"abc", b"d", b"abce")); + assert_eq!(Less, compare_prefixed_slice(b"ab", b"", b"ac")); + assert_eq!(Greater, compare_prefixed_slice(b"a", b"", b"")); + assert_eq!(Greater, compare_prefixed_slice(b"", b"a", b"")); + assert_eq!(Greater, compare_prefixed_slice(b"a", b"a", b"")); + assert_eq!(Greater, compare_prefixed_slice(b"b", b"a", b"a")); + assert_eq!(Greater, compare_prefixed_slice(b"a", b"b", b"a")); + assert_eq!(Greater, compare_prefixed_slice(b"abc", b"xy", b"abcw")); + assert_eq!(Greater, compare_prefixed_slice(b"ab", b"cde", b"a")); + assert_eq!(Greater, compare_prefixed_slice(b"abcd", b"zz", b"abc")); + assert_eq!(Greater, compare_prefixed_slice(b"abc", b"d", b"abc")); + assert_eq!( + Greater, + compare_prefixed_slice(b"aaaa", b"aaab", b"aaaaaaaa") + ); + assert_eq!( + Greater, + compare_prefixed_slice(b"aaaa", b"aaba", b"aaaaaaaa") + ); + assert_eq!(Greater, compare_prefixed_slice(b"abcd", b"x", b"abc")); + + assert_eq!(Less, compare_prefixed_slice(&[0x7F], &[], &[0x80])); + assert_eq!(Greater, compare_prefixed_slice(&[0xFF], &[], &[0x10])); + } +} diff --git a/crates/lsm-tree/src/table/writer/filter/full.rs b/crates/lsm-tree/src/table/writer/filter/full.rs new file mode 100644 index 000000000..83335f09e --- /dev/null +++ b/crates/lsm-tree/src/table/writer/filter/full.rs @@ -0,0 +1,96 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::FilterWriter; +use crate::{ + CompressionType, UserKey, + checksum::ChecksummedWriter, + config::BloomConstructionPolicy, + table::{Block, filter::standard_bloom::Builder}, +}; +use std::{fs::File, io::BufWriter}; + +pub struct FullFilterWriter { + /// Key hashes for AMQ filter + pub bloom_hash_buffer: Vec, + + bloom_policy: BloomConstructionPolicy, +} + +impl FullFilterWriter { + pub fn new(bloom_policy: BloomConstructionPolicy) -> Self { + Self { + bloom_hash_buffer: Vec::new(), + bloom_policy, + } + } +} + +impl FilterWriter for FullFilterWriter { + fn use_partition_size(self: Box, _: u32) -> Box> { + self + } + + fn use_tli_compression(self: Box, _: CompressionType) -> Box> { + self + } + + fn set_filter_policy( + mut self: Box, + policy: BloomConstructionPolicy, + ) -> Box> { + self.bloom_policy = policy; + self + } + + fn register_key(&mut self, key: &UserKey) -> crate::Result<()> { + self.bloom_hash_buffer.push(Builder::get_hash(key)); + Ok(()) + } + + fn finish( + self: Box, + file_writer: &mut sfa::Writer>>, + ) -> crate::Result { + if self.bloom_hash_buffer.is_empty() { + log::trace!("Filter writer has no buffered hashes - not building filter"); + } else { + file_writer.start("filter")?; + + let n = self.bloom_hash_buffer.len(); + + log::trace!( + "Constructing Bloom filter with {n} entries: {:?}", + self.bloom_policy, + ); + + let start = std::time::Instant::now(); + + let filter_bytes = { + let mut builder = self.bloom_policy.init(n); + + for hash in self.bloom_hash_buffer { + builder.set_with_hash(hash); + } + + builder.build() + }; + + log::trace!( + "Built Bloom filter ({}B) in {:?}", + filter_bytes.len(), + start.elapsed(), + ); + + Block::write_into( + file_writer, + &filter_bytes, + crate::table::block::BlockType::Filter, + CompressionType::None, + )?; + } + + Ok(1) + } +} diff --git a/crates/lsm-tree/src/table/writer/filter/mod.rs b/crates/lsm-tree/src/table/writer/filter/mod.rs new file mode 100644 index 000000000..ddf555ab0 --- /dev/null +++ b/crates/lsm-tree/src/table/writer/filter/mod.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod full; +mod partitioned; + +pub use full::FullFilterWriter; +pub use partitioned::PartitionedFilterWriter; + +use crate::{ + CompressionType, UserKey, checksum::ChecksummedWriter, config::BloomConstructionPolicy, +}; +use std::{fs::File, io::BufWriter}; + +pub trait FilterWriter { + // NOTE: We purposefully use a UserKey instead of &[u8] + // so we can clone it without heap allocation, if needed + /// Registers a key in the block index. + fn register_key(&mut self, key: &UserKey) -> crate::Result<()>; + + /// Writes the filter to a file. + /// + /// Returns the number of filter blocks written (always 1 in case of full filter block). + fn finish( + self: Box, + file_writer: &mut sfa::Writer>>, + ) -> crate::Result; + + fn set_filter_policy( + self: Box, + policy: BloomConstructionPolicy, + ) -> Box>; + + fn use_tli_compression( + self: Box, + compression: CompressionType, + ) -> Box>; + + fn use_partition_size(self: Box, size: u32) -> Box>; +} diff --git a/crates/lsm-tree/src/table/writer/filter/partitioned.rs b/crates/lsm-tree/src/table/writer/filter/partitioned.rs new file mode 100644 index 000000000..a2961722c --- /dev/null +++ b/crates/lsm-tree/src/table/writer/filter/partitioned.rs @@ -0,0 +1,209 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::FilterWriter; +use crate::{ + CompressionType, UserKey, + checksum::ChecksummedWriter, + config::BloomConstructionPolicy, + table::{ + Block, BlockHandle, BlockOffset, IndexBlock, KeyedBlockHandle, + block::Header as BlockHeader, filter::standard_bloom::Builder, + }, +}; +use std::{ + fs::File, + io::{BufWriter, Seek, Write}, +}; + +pub struct PartitionedFilterWriter { + final_filter_buffer: Vec, + + tli_handles: Vec, + + /// Key hashes for AMQ filter + pub bloom_hash_buffer: Vec, + approx_filter_size: usize, + + partition_size: u32, + + bloom_policy: BloomConstructionPolicy, + + relative_file_pos: u64, + + last_key: Option, + + compression: CompressionType, +} + +impl PartitionedFilterWriter { + pub fn new(bloom_policy: BloomConstructionPolicy) -> Self { + Self { + final_filter_buffer: Vec::new(), + + bloom_hash_buffer: Vec::new(), + approx_filter_size: 0, + + tli_handles: Vec::new(), + partition_size: 4_096, + bloom_policy, + + relative_file_pos: 0, + + last_key: None, + + compression: CompressionType::None, + } + } + + fn spill_filter_partition(&mut self, key: &UserKey) -> crate::Result<()> { + let filter_bytes = { + let mut builder = self.bloom_policy.init(self.bloom_hash_buffer.len()); + + for hash in self.bloom_hash_buffer.drain(..) { + builder.set_with_hash(hash); + } + + builder.build() + }; + + let header = Block::write_into( + &mut self.final_filter_buffer, + &filter_bytes, + crate::table::block::BlockType::Filter, + CompressionType::None, + )?; + + #[expect( + clippy::cast_possible_truncation, + reason = "data length never gets even close to 4 GiB" + )] + let bytes_written = (header.data_length as usize + BlockHeader::serialized_len()) as u32; + + self.tli_handles.push(KeyedBlockHandle::new( + key.clone(), + 0, + BlockHandle::new(BlockOffset(self.relative_file_pos), bytes_written), + )); + + log::trace!( + "Built Bloom filter partition ({}B) with end_key={key:?} at +{:#X?}", + filter_bytes.len(), + self.relative_file_pos, + ); + + self.bloom_hash_buffer.clear(); + self.approx_filter_size = 0; + self.relative_file_pos += u64::from(bytes_written); + + Ok(()) + } + + fn write_top_level_index( + &mut self, + file_writer: &mut sfa::Writer>>, + index_base_offset: BlockOffset, + ) -> crate::Result<()> { + file_writer.start("filter_tli")?; + + for item in &mut self.tli_handles { + item.shift(index_base_offset); + } + + let mut bytes = vec![]; + IndexBlock::encode_into(&mut bytes, &self.tli_handles)?; + + let header = Block::write_into( + file_writer, + &bytes, + crate::table::block::BlockType::Index, + self.compression, + )?; + + #[expect( + clippy::cast_possible_truncation, + reason = "blocks never even approach u32 size" + )] + let bytes_written = BlockHeader::serialized_len() as u32 + header.data_length; + + debug_assert!(bytes_written > 0, "Top level index should never be empty"); + + log::trace!( + "Written filter top level index, with {} pointers ({bytes_written} bytes) at {index_base_offset:#X?}", + self.tli_handles.len(), + ); + + Ok(()) + } +} + +impl FilterWriter for PartitionedFilterWriter { + fn use_partition_size(mut self: Box, size: u32) -> Box> { + self.partition_size = size; + self + } + + fn use_tli_compression( + mut self: Box, + compression: CompressionType, + ) -> Box> { + self.compression = compression; + self + } + + fn set_filter_policy( + mut self: Box, + policy: BloomConstructionPolicy, + ) -> Box> { + self.bloom_policy = policy; + self + } + + fn register_key(&mut self, key: &UserKey) -> crate::Result<()> { + self.bloom_hash_buffer.push(Builder::get_hash(key)); + + self.approx_filter_size = self + .bloom_policy + .estimated_filter_size(self.bloom_hash_buffer.len()); + + self.last_key = Some(key.clone()); + + if self.approx_filter_size >= self.partition_size as usize { + self.spill_filter_partition(key)?; + } + + Ok(()) + } + + fn finish( + mut self: Box, + file_writer: &mut sfa::Writer>>, + ) -> crate::Result { + if self.last_key.is_none() { + log::trace!("Filter writer has not seen any writes - not building filter"); + return Ok(0); + } + + if !self.bloom_hash_buffer.is_empty() { + #[expect( + clippy::expect_used, + reason = "last key must exist because of initial check" + )] + let last_key = self.last_key.take().expect("last key should exist"); + self.spill_filter_partition(&last_key)?; + } + + let index_base_offset = BlockOffset(file_writer.get_mut().stream_position()?); + + file_writer.start("filter")?; + file_writer.write_all(&self.final_filter_buffer)?; + log::trace!("Concatted filter partitions onto blocks file"); + + let block_count = self.tli_handles.len(); + + self.write_top_level_index(file_writer, index_base_offset)?; + + Ok(block_count) + } +} diff --git a/crates/lsm-tree/src/table/writer/index/full.rs b/crates/lsm-tree/src/table/writer/index/full.rs new file mode 100644 index 000000000..200d04ab3 --- /dev/null +++ b/crates/lsm-tree/src/table/writer/index/full.rs @@ -0,0 +1,86 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + CompressionType, + checksum::ChecksummedWriter, + table::{ + Block, IndexBlock, block::Header as BlockHeader, index_block::KeyedBlockHandle, + writer::index::BlockIndexWriter, + }, +}; +use std::{fs::File, io::BufWriter}; + +pub struct FullIndexWriter { + compression: CompressionType, + block_handles: Vec, +} + +impl FullIndexWriter { + pub fn new() -> Self { + Self { + compression: CompressionType::None, + block_handles: Vec::new(), + } + } +} + +impl BlockIndexWriter for FullIndexWriter { + fn use_partition_size(self: Box, _: u32) -> Box> { + self + } + + fn use_compression( + mut self: Box, + compression: CompressionType, + ) -> Box> { + self.compression = compression; + self + } + + fn register_data_block(&mut self, block_handle: KeyedBlockHandle) -> crate::Result<()> { + log::trace!( + "Registering block at {:?} with size {} [end_key={:?}]", + block_handle.offset(), + block_handle.size(), + block_handle.end_key(), + ); + + self.block_handles.push(block_handle); + + Ok(()) + } + + fn finish( + self: Box, + file_writer: &mut sfa::Writer>>, + ) -> crate::Result { + file_writer.start("tli")?; + + let mut bytes = vec![]; + IndexBlock::encode_into(&mut bytes, &self.block_handles)?; + + let header = Block::write_into( + file_writer, + &bytes, + crate::table::block::BlockType::Index, + self.compression, + )?; + + #[expect( + clippy::cast_possible_truncation, + reason = "blocks never even approach u32 size" + )] + let bytes_written = BlockHeader::serialized_len() as u32 + header.data_length; + + debug_assert!(bytes_written > 0, "Block index should never be empty"); + + log::trace!( + "Written top level index, with {} pointers ({bytes_written}B)", + self.block_handles.len(), + ); + + Ok(1) + } +} diff --git a/crates/lsm-tree/src/table/writer/index/mod.rs b/crates/lsm-tree/src/table/writer/index/mod.rs new file mode 100644 index 000000000..9abc0054b --- /dev/null +++ b/crates/lsm-tree/src/table/writer/index/mod.rs @@ -0,0 +1,32 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod full; +mod partitioned; + +pub use full::FullIndexWriter; +pub use partitioned::PartitionedIndexWriter; + +use crate::{CompressionType, checksum::ChecksummedWriter, table::index_block::KeyedBlockHandle}; +use std::{fs::File, io::BufWriter}; + +pub trait BlockIndexWriter { + /// Registers a data block in the block index. + fn register_data_block(&mut self, block_handle: KeyedBlockHandle) -> crate::Result<()>; + + /// Writes the block index to a file. + /// + /// Returns the number of index blocks written. + fn finish( + self: Box, + file_writer: &mut sfa::Writer>>, + ) -> crate::Result; + + fn use_compression( + self: Box, + compression: CompressionType, + ) -> Box>; + + fn use_partition_size(self: Box, size: u32) -> Box>; +} diff --git a/crates/lsm-tree/src/table/writer/index/partitioned.rs b/crates/lsm-tree/src/table/writer/index/partitioned.rs new file mode 100644 index 000000000..a27c026da --- /dev/null +++ b/crates/lsm-tree/src/table/writer/index/partitioned.rs @@ -0,0 +1,202 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + CompressionType, + checksum::ChecksummedWriter, + table::{ + Block, BlockHandle, BlockOffset, IndexBlock, block::Header as BlockHeader, + index_block::KeyedBlockHandle, writer::index::BlockIndexWriter, + }, +}; +use std::{ + fs::File, + io::{BufWriter, Seek, Write}, +}; + +pub struct PartitionedIndexWriter { + relative_file_pos: u64, + + compression: CompressionType, + + tli_handles: Vec, + data_block_handles: Vec, + + buffer_size: u32, + partition_size: u32, + + index_block_count: usize, + + block_buffer: Vec, + + final_write_buffer: Vec, +} + +impl PartitionedIndexWriter { + pub fn new() -> Self { + Self { + relative_file_pos: 0, + buffer_size: 0, + index_block_count: 0, + + partition_size: 4_096, + compression: CompressionType::None, + + tli_handles: Vec::new(), + data_block_handles: Vec::new(), + block_buffer: Vec::with_capacity(4_096), + + final_write_buffer: Vec::new(), + } + } + + fn cut_index_block(&mut self) -> crate::Result<()> { + let mut bytes = vec![]; + IndexBlock::encode_into(&mut bytes, &self.data_block_handles)?; + + let header = Block::write_into( + &mut self.block_buffer, + &bytes, + crate::table::block::BlockType::Index, + self.compression, + )?; + + #[expect( + clippy::cast_possible_truncation, + reason = "blocks never even approach size of 4 GiB" + )] + let bytes_written = BlockHeader::serialized_len() as u32 + header.data_length; + + // Also, we are allowed to remove the last item + // to get ownership of it, because the chunk is cleared after + // this anyway + #[expect(clippy::expect_used, reason = "chunk is not empty")] + let last = self + .data_block_handles + .pop() + .expect("Chunk should not be empty"); + + let index_block_handle = KeyedBlockHandle::new( + last.end_key().clone(), + last.seqno(), + BlockHandle::new(BlockOffset(self.relative_file_pos), bytes_written), + ); + + log::trace!( + "Built Bloom filter partition ({bytes_written}B) with end_key={:?} at +{:#X?}", + last.end_key(), + self.relative_file_pos, + ); + + self.tli_handles.push(index_block_handle); + self.final_write_buffer.append(&mut self.block_buffer); + + // Adjust metadata + self.index_block_count += 1; + self.relative_file_pos += u64::from(bytes_written); + + // IMPORTANT: Clear buffer after everything else + self.data_block_handles.clear(); + self.buffer_size = 0; + + Ok(()) + } + + fn write_top_level_index( + &mut self, + file_writer: &mut sfa::Writer>>, + index_base_offset: BlockOffset, + ) -> crate::Result<()> { + file_writer.start("tli")?; + + for item in &mut self.tli_handles { + item.shift(index_base_offset); + } + + let mut bytes = vec![]; + IndexBlock::encode_into(&mut bytes, &self.tli_handles)?; + + let header = Block::write_into( + file_writer, + &bytes, + crate::table::block::BlockType::Index, + self.compression, + )?; + + #[expect( + clippy::cast_possible_truncation, + reason = "blocks never even approach 4 GiB in size" + )] + let bytes_written = BlockHeader::serialized_len() as u32 + header.data_length; + + debug_assert!(bytes_written > 0, "Top level index should never be empty"); + + log::trace!( + "Written top level index, with {} pointers ({bytes_written} bytes)", + self.tli_handles.len(), + ); + + Ok(()) + } +} + +impl BlockIndexWriter for PartitionedIndexWriter { + fn use_partition_size(mut self: Box, size: u32) -> Box> { + self.partition_size = size; + self + } + + fn use_compression( + mut self: Box, + compression: CompressionType, + ) -> Box> { + self.compression = compression; + self + } + + fn register_data_block(&mut self, block_handle: KeyedBlockHandle) -> crate::Result<()> { + log::trace!( + "Registering block at {:?} with size {} [end_key={:?}]", + block_handle.offset(), + block_handle.size(), + block_handle.end_key(), + ); + + #[expect( + clippy::cast_possible_truncation, + reason = "key is u16 max, so we can not exceed u32::MAX" + )] + let block_handle_size = + (block_handle.end_key().len() + std::mem::size_of::()) as u32; + + self.buffer_size += block_handle_size; + + self.data_block_handles.push(block_handle); + + if self.buffer_size >= self.partition_size { + self.cut_index_block()?; + } + + Ok(()) + } + + fn finish( + mut self: Box, + file_writer: &mut sfa::Writer>>, + ) -> crate::Result { + if self.buffer_size > 0 { + self.cut_index_block()?; + } + + let index_base_offset = BlockOffset(file_writer.get_mut().stream_position()?); + + file_writer.start("index")?; + file_writer.write_all(&self.final_write_buffer)?; + log::trace!("Concatted index partitions onto blocks file"); + + self.write_top_level_index(file_writer, index_base_offset)?; + + Ok(self.index_block_count) + } +} diff --git a/crates/lsm-tree/src/table/writer/meta.rs b/crates/lsm-tree/src/table/writer/meta.rs new file mode 100644 index 000000000..bdeed4916 --- /dev/null +++ b/crates/lsm-tree/src/table/writer/meta.rs @@ -0,0 +1,65 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{SeqNo, UserKey, table::BlockOffset}; + +pub struct Metadata { + /// Written data block count + pub data_block_count: usize, + + /// Written item count + pub item_count: usize, + + /// Tombstone count + pub tombstone_count: usize, + + /// Weak tombstone (single delete) count + pub weak_tombstone_count: usize, + + /// Weak tombstone + value pairs that become reclaimable when GC watermark advances + pub weak_tombstone_reclaimable_count: usize, + + /// Written key count (unique keys) + pub key_count: usize, + + /// Current file position of writer + pub file_pos: BlockOffset, + + /// Only takes user data into account + pub uncompressed_size: u64, + + /// First encountered key + pub first_key: Option, + + /// Last encountered key + pub last_key: Option, + + /// Lowest encountered seqno + pub lowest_seqno: SeqNo, + + /// Highest encountered seqno + pub highest_seqno: SeqNo, +} + +impl Default for Metadata { + fn default() -> Self { + Self { + data_block_count: 0, + + item_count: 0, + tombstone_count: 0, + weak_tombstone_count: 0, + weak_tombstone_reclaimable_count: 0, + key_count: 0, + file_pos: BlockOffset(0), + uncompressed_size: 0, + + first_key: None, + last_key: None, + + lowest_seqno: SeqNo::MAX, + highest_seqno: 0, + } + } +} diff --git a/crates/lsm-tree/src/table/writer/mod.rs b/crates/lsm-tree/src/table/writer/mod.rs new file mode 100644 index 000000000..f7636c5aa --- /dev/null +++ b/crates/lsm-tree/src/table/writer/mod.rs @@ -0,0 +1,678 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod filter; +mod index; +mod meta; + +use super::{ + Block, BlockOffset, DataBlock, KeyedBlockHandle, block::Header as BlockHeader, + filter::BloomConstructionPolicy, +}; +use crate::{ + Checksum, CompressionType, InternalValue, TableId, UserKey, ValueType, + checksum::{ChecksumType, ChecksummedWriter}, + coding::Encode, + file::fsync_directory, + table::{ + BlockHandle, + writer::{ + filter::{FilterWriter, FullFilterWriter}, + index::FullIndexWriter, + }, + }, + time::unix_timestamp, + vlog::BlobFileId, +}; +use index::BlockIndexWriter; +use std::{fs::File, io::BufWriter, path::PathBuf}; + +#[derive(Copy, Clone, PartialEq, Eq, Debug, std::hash::Hash)] +pub struct LinkedFile { + pub blob_file_id: BlobFileId, + pub bytes: u64, + pub on_disk_bytes: u64, + pub len: usize, +} + +#[derive(Default)] +struct FixedLen { + len: Option, + mixed: bool, +} + +impl FixedLen { + fn observe(&mut self, len: usize) { + if self.mixed { + return; + } + + match self.len { + None => self.len = Some(len), + Some(previous) if previous != len => self.mixed = true, + Some(_) => {} + } + } + + fn get(&self) -> Option { + (!self.mixed).then_some(self.len).flatten() + } +} + +/// Serializes and compresses values into blocks and writes them to disk as a table +pub struct Writer { + /// Table file path + pub(crate) path: PathBuf, + + table_id: TableId, + + data_block_restart_interval: u8, + index_block_restart_interval: u8, + + meta_partition_size: u32, + + data_block_size: u32, + + data_block_hash_ratio: f32, + + /// Compression to use for data blocks + data_block_compression: CompressionType, + + /// Compression to use for data blocks + index_block_compression: CompressionType, + + /// Buffer to serialize blocks into + block_buffer: Vec, + + /// File writer + #[expect(clippy::struct_field_names)] + file_writer: sfa::Writer>>, + + /// Writer of index blocks + #[expect(clippy::struct_field_names)] + index_writer: Box>>, + + /// Writer of filter + #[expect(clippy::struct_field_names)] + filter_writer: Box>>, + + /// Buffer of KVs + chunk: Vec, + chunk_size: usize, + chunk_key_len: FixedLen, + chunk_value_len: FixedLen, + + pub(crate) meta: meta::Metadata, + + /// Stores the previous block position (used for creating back links) + prev_pos: (BlockOffset, BlockOffset), + + current_key: Option, + + bloom_policy: BloomConstructionPolicy, + + /// Tracks the previously written item to detect weak tombstone/value pairs + previous_item: Option<(UserKey, ValueType)>, + + linked_blob_files: Vec, + + initial_level: u8, +} + +impl Writer { + pub fn new(path: PathBuf, table_id: TableId, initial_level: u8) -> crate::Result { + let writer = BufWriter::with_capacity(u16::MAX.into(), File::create_new(&path)?); + let writer = ChecksummedWriter::new(writer); + let mut writer = sfa::Writer::from_writer(writer); + writer.start("data")?; + + Ok(Self { + initial_level, + + meta: meta::Metadata::default(), + + table_id, + + data_block_restart_interval: 16, + index_block_restart_interval: 1, + + data_block_hash_ratio: 0.0, + + meta_partition_size: 4_096, + + data_block_size: 4_096, + + data_block_compression: CompressionType::None, + index_block_compression: CompressionType::None, + + path: std::path::absolute(path)?, + + index_writer: Box::new(FullIndexWriter::new()), + filter_writer: Box::new(FullFilterWriter::new(BloomConstructionPolicy::default())), + + block_buffer: Vec::new(), + file_writer: writer, + chunk: Vec::new(), + + prev_pos: (BlockOffset(0), BlockOffset(0)), + + chunk_size: 0, + chunk_key_len: FixedLen::default(), + chunk_value_len: FixedLen::default(), + + current_key: None, + + bloom_policy: BloomConstructionPolicy::default(), + + previous_item: None, + + linked_blob_files: Vec::new(), + }) + } + + pub fn link_blob_file( + &mut self, + blob_file_id: BlobFileId, + len: usize, + bytes: u64, + on_disk_bytes: u64, + ) { + self.linked_blob_files.push(LinkedFile { + blob_file_id, + bytes, + on_disk_bytes, + len, + }); + } + + #[must_use] + pub fn use_partitioned_filter(mut self) -> Self { + self.filter_writer = Box::new(filter::PartitionedFilterWriter::new(self.bloom_policy)) + .use_tli_compression(self.index_block_compression); + self + } + + #[must_use] + pub fn use_partitioned_index(mut self) -> Self { + self.index_writer = Box::new(index::PartitionedIndexWriter::new()) + .use_compression(self.index_block_compression); + self + } + + #[must_use] + pub fn use_data_block_restart_interval(mut self, interval: u8) -> Self { + self.data_block_restart_interval = interval; + self + } + + #[must_use] + pub fn use_index_block_restart_interval(mut self, interval: u8) -> Self { + self.index_block_restart_interval = interval; + self + } + + #[must_use] + pub fn use_data_block_hash_ratio(mut self, ratio: f32) -> Self { + self.data_block_hash_ratio = ratio; + self + } + + #[must_use] + pub fn use_data_block_size(mut self, size: u32) -> Self { + assert!( + size <= 4 * 1_024 * 1_024, + "data block size must be <= 4 MiB", + ); + self.data_block_size = size; + self + } + + #[must_use] + pub fn use_meta_partition_size(mut self, size: u32) -> Self { + assert!( + size <= 4 * 1_024 * 1_024, + "data block size must be <= 4 MiB", + ); + self.meta_partition_size = size; + self.index_writer = self.index_writer.use_partition_size(size); + self.filter_writer = self.filter_writer.use_partition_size(size); + self + } + + #[must_use] + pub fn use_data_block_compression(mut self, compression: CompressionType) -> Self { + self.data_block_compression = compression; + self + } + + #[must_use] + pub fn use_index_block_compression(mut self, compression: CompressionType) -> Self { + self.index_block_compression = compression; + self.index_writer = self.index_writer.use_compression(compression); + self.filter_writer = self.filter_writer.use_tli_compression(compression); + self + } + + #[must_use] + pub fn use_bloom_policy(mut self, bloom_policy: BloomConstructionPolicy) -> Self { + self.bloom_policy = bloom_policy; + self.filter_writer = self.filter_writer.set_filter_policy(bloom_policy); + self + } + + /// Writes an item. + /// + /// # Note + /// + /// It's important that the incoming stream of items is correctly + /// sorted as described by the [`UserKey`], otherwise the block layout will + /// be non-sense. + pub fn write(&mut self, item: InternalValue) -> crate::Result<()> { + let value_type = item.key.value_type; + let seqno = item.key.seqno; + let user_key = item.key.user_key.clone(); + let value_len = item.value.len(); + + if item.is_tombstone() { + self.meta.tombstone_count += 1; + } + + if value_type == ValueType::WeakTombstone { + self.meta.weak_tombstone_count += 1; + } + + if value_type == ValueType::Value { + if let Some((prev_key, prev_type)) = &self.previous_item { + if prev_type == &ValueType::WeakTombstone && prev_key.as_ref() == user_key.as_ref() + { + self.meta.weak_tombstone_reclaimable_count += 1; + } + } + } + + // NOTE: Check if we visit a new key + if Some(&user_key) != self.current_key.as_ref() { + self.meta.key_count += 1; + self.current_key = Some(user_key.clone()); + + // IMPORTANT: Do not buffer *every* item's key + // because there may be multiple versions + // of the same key + + if self.bloom_policy.is_active() { + self.filter_writer.register_key(&user_key)?; + } + } + + if self.meta.first_key.is_none() { + self.meta.first_key = Some(user_key.clone()); + } + + self.chunk_size += user_key.len() + value_len; + self.chunk_key_len.observe(user_key.len()); + if !value_type.is_tombstone() { + self.chunk_value_len.observe(value_len); + } + self.chunk.push(item); + self.previous_item = Some((user_key, value_type)); + + if self.chunk_size >= self.data_block_size as usize { + self.spill_block()?; + } + + self.meta.lowest_seqno = self.meta.lowest_seqno.min(seqno); + self.meta.highest_seqno = self.meta.highest_seqno.max(seqno); + + Ok(()) + } + + pub(crate) fn write_distinct(&mut self, item: InternalValue) -> crate::Result<()> { + let value_type = item.key.value_type; + let seqno = item.key.seqno; + let user_key = &item.key.user_key; + let value_len = item.value.len(); + + if item.is_tombstone() { + self.meta.tombstone_count += 1; + } + + if value_type == ValueType::WeakTombstone { + self.meta.weak_tombstone_count += 1; + } + + self.meta.key_count += 1; + + if self.bloom_policy.is_active() { + self.filter_writer.register_key(user_key)?; + } + + if self.meta.first_key.is_none() { + self.meta.first_key = Some(user_key.clone()); + } + + self.chunk_size += user_key.len() + value_len; + self.chunk_key_len.observe(user_key.len()); + if !value_type.is_tombstone() { + self.chunk_value_len.observe(value_len); + } + self.chunk.push(item); + + if self.chunk_size >= self.data_block_size as usize { + self.spill_block()?; + } + + self.meta.lowest_seqno = self.meta.lowest_seqno.min(seqno); + self.meta.highest_seqno = self.meta.highest_seqno.max(seqno); + + Ok(()) + } + + /// Writes a compressed block to disk. + /// + /// This is triggered when a `Writer::write` causes the buffer to grow to the configured `block_size`. + /// + /// Should only be called when the block has items in it. + pub(crate) fn spill_block(&mut self) -> crate::Result<()> { + let Some(last) = self.chunk.last() else { + return Ok(()); + }; + + self.block_buffer.clear(); + + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + let fixed_value_len = self.chunk_value_len.get().map(|len| len as u32); + // With compressed empty-value blocks, keeping the repeated key-length byte gives LZ4 + // a useful alignment/pattern and is smaller than omitting both lengths. + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 length max")] + let fixed_key_len = self + .chunk_key_len + .get() + .filter(|_| { + self.data_block_compression == CompressionType::None || fixed_value_len != Some(0) + }) + .map(|len| len as u16); + + DataBlock::encode_into_with_fixed_lengths( + &mut self.block_buffer, + &self.chunk, + self.data_block_restart_interval, + self.data_block_hash_ratio, + fixed_key_len, + fixed_value_len, + )?; + + let header = Block::write_into( + &mut self.file_writer, + &self.block_buffer, + super::block::BlockType::Data, + self.data_block_compression, + )?; + + self.meta.uncompressed_size += u64::from(header.uncompressed_length); + + #[expect( + clippy::cast_possible_truncation, + reason = "block header is a couple of bytes only, so cast is fine" + )] + let bytes_written = BlockHeader::serialized_len() as u32 + header.data_length; + + self.index_writer + .register_data_block(KeyedBlockHandle::new( + last.key.user_key.clone(), + last.key.seqno, + BlockHandle::new(self.meta.file_pos, bytes_written), + ))?; + + // Adjust metadata + self.meta.file_pos += u64::from(bytes_written); + self.meta.item_count += self.chunk.len(); + self.meta.data_block_count += 1; + + // Back link stuff + self.prev_pos.0 = self.prev_pos.1; + self.prev_pos.1 += u64::from(bytes_written); + + // Set last key + self.meta.last_key = Some( + // NOTE: We are allowed to remove the last item + // to get ownership of it, because the chunk is cleared after + // this anyway + #[expect(clippy::expect_used, reason = "chunk is not empty")] + self.chunk + .pop() + .expect("chunk should not be empty") + .key + .user_key, + ); + + // IMPORTANT: Clear chunk after everything else + self.chunk.clear(); + self.chunk_size = 0; + self.chunk_key_len = FixedLen::default(); + self.chunk_value_len = FixedLen::default(); + + Ok(()) + } + + // TODO: split meta writing into new function + #[expect(clippy::too_many_lines)] + /// Finishes the table, making sure all data is written durably + pub fn finish(mut self) -> crate::Result> { + use std::io::Write; + + self.spill_block()?; + + // No items written! Just delete table file and return nothing + if self.meta.item_count == 0 { + std::fs::remove_file(&self.path)?; + return Ok(None); + } + + // Write index + log::trace!("Finishing index writer"); + let index_block_count = self.index_writer.finish(&mut self.file_writer)?; + + // Write filter + log::trace!("Finishing filter writer"); + let filter_block_count = self.filter_writer.finish(&mut self.file_writer)?; + + if !self.linked_blob_files.is_empty() { + use byteorder::{LE, WriteBytesExt}; + + self.file_writer.start("linked_blob_files")?; + + #[expect( + clippy::cast_possible_truncation, + reason = "there are never 4 billion blob files linked to a single table" + )] + self.file_writer + .write_u32::(self.linked_blob_files.len() as u32)?; + + for file in self.linked_blob_files { + self.file_writer.write_u64::(file.blob_file_id)?; + self.file_writer.write_u64::(file.len as u64)?; + self.file_writer.write_u64::(file.bytes)?; + self.file_writer.write_u64::(file.on_disk_bytes)?; + } + } + + self.file_writer.start("table_version")?; + self.file_writer.write_all(&[0x4])?; + + // Write metadata + self.file_writer.start("meta")?; + + { + fn meta(key: &str, value: &[u8]) -> InternalValue { + InternalValue::from_components(key, value, 0, crate::ValueType::Value) + } + + let meta_items = [ + meta( + "block_count#data", + &(self.meta.data_block_count as u64).to_le_bytes(), + ), + meta( + "block_count#filter", + &(filter_block_count as u64).to_le_bytes(), + ), + meta( + "block_count#index", + &(index_block_count as u64).to_le_bytes(), + ), + meta("checksum_type", &[u8::from(ChecksumType::Xxh3)]), + meta( + "compression#data", + &self.data_block_compression.encode_into_vec(), + ), + meta( + "compression#index", + &self.index_block_compression.encode_into_vec(), + ), + meta("crate_version", env!("CARGO_PKG_VERSION").as_bytes()), + meta("created_at", &unix_timestamp().as_nanos().to_le_bytes()), + meta( + "data_block_hash_ratio", + &self.data_block_hash_ratio.to_le_bytes(), + ), + meta("file_size", &self.meta.file_pos.to_le_bytes()), + meta("filter_hash_type", &[u8::from(ChecksumType::Xxh3)]), + meta("index_keys_have_seqno", &[0x1]), + meta("initial_level", &self.initial_level.to_le_bytes()), + meta("item_count", &(self.meta.item_count as u64).to_le_bytes()), + meta( + "key#max", + // NOTE: At the beginning we check that we have written at least 1 item, so last_key must exist + #[expect(clippy::expect_used)] + self.meta.last_key.as_ref().expect("should exist"), + ), + meta( + "key#min", + // NOTE: At the beginning we check that we have written at least 1 item, so first_key must exist + #[expect(clippy::expect_used)] + self.meta.first_key.as_ref().expect("should exist"), + ), + meta("key_count", &(self.meta.key_count as u64).to_le_bytes()), + meta("prefix_truncation#data", &[1]), // NOTE: currently prefix truncation can not be disabled + meta("prefix_truncation#index", &[1]), // NOTE: currently prefix truncation can not be disabled + meta( + "restart_interval#data", + &self.data_block_restart_interval.to_le_bytes(), + ), + meta( + "restart_interval#index", + &self.index_block_restart_interval.to_le_bytes(), + ), + meta("seqno#max", &self.meta.highest_seqno.to_le_bytes()), + meta("seqno#min", &self.meta.lowest_seqno.to_le_bytes()), + meta("table_id", &self.table_id.to_le_bytes()), + meta("table_version", &[4u8]), + meta( + "tombstone_count", + &(self.meta.tombstone_count as u64).to_le_bytes(), + ), + meta("user_data_size", &self.meta.uncompressed_size.to_le_bytes()), + meta( + "weak_tombstone_count", + &(self.meta.weak_tombstone_count as u64).to_le_bytes(), + ), + meta( + "weak_tombstone_reclaimable", + &(self.meta.weak_tombstone_reclaimable_count as u64).to_le_bytes(), + ), + ]; + + // NOTE: Just to make sure the items are definitely sorted + #[cfg(debug_assertions)] + { + let is_sorted = meta_items.iter().is_sorted_by_key(|kv| &kv.key); + assert!(is_sorted, "meta items not sorted correctly"); + } + + self.block_buffer.clear(); + + // TODO: disable binary index: https://github.com/fjall-rs/lsm-tree/issues/185 + DataBlock::encode_into(&mut self.block_buffer, &meta_items, 1, 0.0)?; + + Block::write_into( + &mut self.file_writer, + &self.block_buffer, + crate::table::block::BlockType::Meta, + CompressionType::None, + )?; + }; + + // Write fixed-size trailer + // and flush & fsync the table file + let mut checksum = self.file_writer.into_inner()?; + checksum.inner_mut().get_mut().sync_all()?; + let checksum = checksum.checksum(); + + // IMPORTANT: fsync folder on Unix + + #[expect( + clippy::expect_used, + reason = "if there's no parent folder, something has gone horribly wrong" + )] + fsync_directory(self.path.parent().expect("should have folder"))?; + + log::debug!( + "Written {} items in {} blocks into new table file #{}, written {} MiB", + self.meta.item_count, + self.meta.data_block_count, + self.table_id, + *self.meta.file_pos / 1_024 / 1_024, + ); + + Ok(Some((self.table_id, checksum))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn table_writer_count() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("1"); + let mut writer = Writer::new(path, 1, 0)?; + + assert_eq!(0, writer.meta.key_count); + assert_eq!(0, writer.chunk_size); + + writer.write(InternalValue::from_components( + b"a", + b"a", + 0, + ValueType::Value, + ))?; + assert_eq!(1, writer.meta.key_count); + assert_eq!(2, writer.chunk_size); + + writer.write(InternalValue::from_components( + b"b", + b"b", + 0, + ValueType::Value, + ))?; + assert_eq!(2, writer.meta.key_count); + assert_eq!(4, writer.chunk_size); + + writer.write(InternalValue::from_components( + b"c", + b"c", + 0, + ValueType::Value, + ))?; + assert_eq!(3, writer.meta.key_count); + assert_eq!(6, writer.chunk_size); + + writer.spill_block()?; + assert_eq!(0, writer.chunk_size); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/time.rs b/crates/lsm-tree/src/time.rs new file mode 100644 index 000000000..4cb0f3845 --- /dev/null +++ b/crates/lsm-tree/src/time.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Gets the unix timestamp as a duration +pub fn unix_timestamp() -> std::time::Duration { + #[cfg(test)] + #[allow(clippy::significant_drop_in_scrutinee, clippy::expect_used)] + { + if let Some(cell) = NOW_OVERRIDE.get() { + if let Some(override_val) = *cell.lock().expect("lock is poisoned") { + return override_val; + } + } + } + + let now = std::time::SystemTime::now(); + + #[expect(clippy::expect_used, reason = "trivial")] + now.duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("time went backwards") +} + +#[cfg(test)] +use std::sync::{Mutex, OnceLock}; + +#[cfg(test)] +static NOW_OVERRIDE: OnceLock>> = OnceLock::new(); + +#[cfg(test)] +#[allow(clippy::expect_used)] +pub fn set_unix_timestamp_for_test(value: Option) { + let cell = NOW_OVERRIDE.get_or_init(|| Mutex::new(None)); + *cell.lock().expect("lock is poisoned") = value; +} diff --git a/crates/lsm-tree/src/tree/ingest.rs b/crates/lsm-tree/src/tree/ingest.rs new file mode 100644 index 000000000..5dc5a9f37 --- /dev/null +++ b/crates/lsm-tree/src/tree/ingest.rs @@ -0,0 +1,387 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::Tree; +use crate::{ + BlobIndirection, SeqNo, UserKey, UserValue, config::FilterPolicyEntry, + table::multi_writer::MultiWriter, +}; +use std::path::PathBuf; + +pub const INITIAL_CANONICAL_LEVEL: usize = 1; + +/// Bulk ingestion +/// +/// Items NEED to be added in ascending key order. +/// +/// Ingested data bypasses memtables and is written directly into new tables, +/// using the same table writer configuration that is used for flush and compaction. +pub struct Ingestion<'a> { + folder: PathBuf, + tree: &'a Tree, + pub(crate) writer: MultiWriter, + seqno: SeqNo, + last_key: Option, +} + +impl<'a> Ingestion<'a> { + /// Creates a new ingestion. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn new(tree: &'a Tree) -> crate::Result { + let folder = tree.config.path.join(crate::file::TABLES_FOLDER); + log::debug!("Ingesting into tables in {}", folder.display()); + + let index_partitioning = tree + .config + .index_block_partitioning_policy + .get(INITIAL_CANONICAL_LEVEL); + + let filter_partitioning = tree + .config + .filter_block_partitioning_policy + .get(INITIAL_CANONICAL_LEVEL); + + // TODO: maybe create a PrepareMultiWriter that can be used by flush, ingest and compaction worker + let mut writer = MultiWriter::new( + folder.clone(), + tree.table_id_counter.clone(), + 64 * 1_024 * 1_024, + 6, + )? + .use_bloom_policy({ + if tree.config.expect_point_read_hits { + crate::config::BloomConstructionPolicy::BitsPerKey(0.0) + } else if let FilterPolicyEntry::Bloom(p) = + tree.config.filter_policy.get(INITIAL_CANONICAL_LEVEL) + { + p + } else { + crate::config::BloomConstructionPolicy::BitsPerKey(0.0) + } + }) + .use_data_block_size( + tree.config + .data_block_size_policy + .get(INITIAL_CANONICAL_LEVEL), + ) + .use_data_block_hash_ratio( + tree.config + .data_block_hash_ratio_policy + .get(INITIAL_CANONICAL_LEVEL), + ) + .use_data_block_compression( + tree.config + .data_block_compression_policy + .get(INITIAL_CANONICAL_LEVEL), + ) + .use_index_block_compression( + tree.config + .index_block_compression_policy + .get(INITIAL_CANONICAL_LEVEL), + ) + .use_data_block_restart_interval( + tree.config + .data_block_restart_interval_policy + .get(INITIAL_CANONICAL_LEVEL), + ) + .use_index_block_restart_interval( + tree.config + .index_block_restart_interval_policy + .get(INITIAL_CANONICAL_LEVEL), + ); + + if index_partitioning { + writer = writer.use_partitioned_index(); + } + if filter_partitioning { + writer = writer.use_partitioned_filter(); + } + + Ok(Self { + folder, + tree, + writer, + seqno: 0, + last_key: None, + }) + } + + /// Writes a key-value pair. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub(crate) fn write_indirection( + &mut self, + key: UserKey, + indirection: BlobIndirection, + ) -> crate::Result<()> { + use crate::coding::Encode; + + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + let cloned_key = key.clone(); + self.writer + .write_distinct(crate::InternalValue::from_components( + key, + indirection.encode_into_vec(), + self.seqno, + crate::ValueType::Indirection, + ))?; + + self.writer.register_blob(indirection); + + // Remember the last user key to validate the next call's ordering + self.last_key = Some(cloned_key); + + Ok(()) + } + + /// Writes a key-value pair. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write(&mut self, key: UserKey, value: UserValue) -> crate::Result<()> { + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + self.writer + .write_distinct(crate::InternalValue::from_components( + key.clone(), + value, + self.seqno, + crate::ValueType::Value, + ))?; + + // Remember the last user key to validate the next call's ordering + self.last_key = Some(key); + + Ok(()) + } + + /// Writes a key that the caller has already proven is greater than every + /// preceding key. Debug builds retain the ordering assertion. + #[doc(hidden)] + pub fn write_prevalidated(&mut self, key: UserKey, value: UserValue) -> crate::Result<()> { + #[cfg(debug_assertions)] + { + if let Some(prev) = &self.last_key { + debug_assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + self.last_key = Some(key.clone()); + } + + self.writer + .write_distinct(crate::InternalValue::from_components( + key, + value, + self.seqno, + crate::ValueType::Value, + )) + } + + /// Writes a tombstone for a key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write_tombstone(&mut self, key: UserKey) -> crate::Result<()> { + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + self.writer + .write_distinct(crate::InternalValue::from_components( + key.clone(), + crate::UserValue::empty(), + self.seqno, + crate::ValueType::Tombstone, + ))?; + + // Remember the last user key to validate the next call's ordering + self.last_key = Some(key); + + Ok(()) + } + + /// Writes a weak tombstone for a key. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write_weak_tombstone(&mut self, key: UserKey) -> crate::Result<()> { + if let Some(prev) = &self.last_key { + assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + + self.writer + .write_distinct(crate::InternalValue::from_components( + key.clone(), + crate::UserValue::empty(), + self.seqno, + crate::ValueType::WeakTombstone, + ))?; + + // Remember the last user key to validate the next call's ordering + self.last_key = Some(key); + + Ok(()) + } + + /// Writes a weak tombstone whose key the caller has already proven is + /// greater than every preceding key. Debug builds retain the assertion. + #[doc(hidden)] + pub fn write_prevalidated_weak_tombstone(&mut self, key: UserKey) -> crate::Result<()> { + #[cfg(debug_assertions)] + { + if let Some(prev) = &self.last_key { + debug_assert!( + key > *prev, + "next key in ingestion must be greater than last key" + ); + } + self.last_key = Some(key.clone()); + } + + self.writer + .write_distinct(crate::InternalValue::from_components( + key, + crate::UserValue::empty(), + self.seqno, + crate::ValueType::WeakTombstone, + )) + } + + /// Finishes the ingestion. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[allow(clippy::significant_drop_tightening)] + pub fn finish(self) -> crate::Result<()> { + use crate::{AbstractTree, Table}; + + if self.writer.is_empty() { + log::trace!("No data written to Ingestion, returning early"); + return Ok(()); + } + + // CRITICAL SECTION: Atomic flush + seqno allocation + registration + // + // We must ensure no concurrent writes interfere between flushing the + // active memtable and registering the ingested tables. The sequence is: + // 1. Acquire flush lock (prevents concurrent flushes) + // 2. Flush active memtable (ensures no pending writes) + // 3. Finish ingestion writer (creates table files) + // 4. Allocate next global seqno (atomic timestamp) + // 5. Recover tables with that seqno + // 6. Register version with same seqno + // + // Why not flush in new()? + // If we flushed in new(), there would be a race condition: + // new() -> flush -> [TIME PASSES + OTHER WRITES] -> finish() -> seqno + // The seqno would be disconnected from the flush, violating MVCC. + // + // By holding the flush lock throughout, we guarantee atomicity. + let flush_lock = self.tree.get_flush_lock(); + + // Flush any pending memtable writes to ensure ingestion sees a + // consistent snapshot and lookup order remains correct. + // We call rotate + flush directly because we already hold the lock. + self.tree.rotate_memtable(); + self.tree.flush(&flush_lock, 0)?; + + // Finalize the ingestion writer, writing all buffered data to disk. + let results = self.writer.finish()?; + + log::info!("Finished ingestion writer"); + + // Acquire locks for version registration. We must hold both the + // compaction state lock and version history lock to safely modify + // the tree's version. + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut _compaction_state = self.tree.compaction_state.lock().expect("lock is poisoned"); + + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_lock = self.tree.version_history.write().expect("lock is poisoned"); + + // Allocate the next global sequence number. This seqno will be shared + // by all ingested tables and the version that registers them, ensuring + // consistent MVCC snapshots. + let global_seqno = self.tree.config.seqno.next(); + + // Recover all created tables, assigning them the global_seqno we just + // allocated. This ensures all ingested tables share the same sequence + // number, which is critical for MVCC correctness. + // + // We intentionally do NOT pin filter/index blocks here. Large ingests + // are typically placed in level 1, and pinning would increase memory + // pressure unnecessarily. + let created_tables = results + .into_iter() + .map(|(table_id, checksum)| -> crate::Result
{ + Table::recover( + self.folder.join(table_id.to_string()), + checksum, + global_seqno, + self.tree.id, + self.tree.config.cache.clone(), + self.tree.config.descriptor_table.clone(), + false, + false, + #[cfg(feature = "metrics")] + self.tree.metrics.clone(), + ) + }) + .collect::>>()?; + + // Upgrade the version with our ingested tables, using the global_seqno + // we allocated earlier. This ensures the version and all tables share + // the same sequence number. + // + // We use upgrade_version_with_seqno (instead of upgrade_version) because + // we need precise control over the seqno: it must match the seqno we + // already assigned to the recovered tables. + version_lock.upgrade_version_with_seqno( + &self.tree.config.path, + |current| { + let mut copy = current.clone(); + copy.version = copy.version.with_new_l0_run(&created_tables, None, None); + Ok(copy) + }, + global_seqno, + &self.tree.config.visible_seqno, + )?; + + // Perform maintenance on the version history (e.g., clean up old versions). + // We use gc_watermark=0 since ingestion doesn't affect sealed memtables. + if let Err(e) = version_lock.maintenance(&self.tree.config.path, 0) { + log::warn!("Version GC failed: {e:?}"); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/tree/inner.rs b/crates/lsm-tree/src/tree/inner.rs new file mode 100644 index 000000000..85c8f9d0e --- /dev/null +++ b/crates/lsm-tree/src/tree/inner.rs @@ -0,0 +1,115 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + SequenceNumberCounter, TableId, + compaction::state::CompactionState, + config::Config, + stop_signal::StopSignal, + version::{SuperVersions, Version, persist_version}, +}; +use std::sync::{Arc, Mutex, RwLock, atomic::AtomicU64}; + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +/// Unique tree ID +/// +/// Tree IDs are monotonically increasing integers. +pub type TreeId = u64; + +/// Unique memtable ID +/// +/// Memtable IDs map one-to-one to some table. +pub type MemtableId = u64; + +/// Hands out a unique (monotonically increasing) tree ID. +pub fn get_next_tree_id() -> TreeId { + static TREE_ID_COUNTER: AtomicU64 = AtomicU64::new(0); + TREE_ID_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) +} + +pub struct TreeInner { + /// Unique tree ID + pub id: TreeId, + + /// Hands out a unique (monotonically increasing) memtable ID + #[doc(hidden)] + pub memtable_id_counter: SequenceNumberCounter, + + /// Hands out a unique (monotonically increasing) table ID + #[doc(hidden)] + pub table_id_counter: SequenceNumberCounter, + + // This is not really used in the normal tree, but we need it in the blob tree + /// Hands out a unique (monotonically increasing) blob file ID + pub(crate) blob_file_id_counter: SequenceNumberCounter, + + pub(crate) version_history: Arc>, + + pub(crate) compaction_state: Arc>, + + /// Tree configuration + pub config: Arc, + + /// Compaction may take a while; setting the signal to `true` + /// will interrupt the compaction and kill the worker. + pub(crate) stop_signal: StopSignal, + + /// Used by major compaction to be the exclusive compaction going on. + /// + /// Minor compactions use `major_compaction_lock.read()` instead, so they + /// can be concurrent next to each other. + pub(crate) major_compaction_lock: RwLock<()>, + + /// Serializes flush operations. + pub(crate) flush_lock: Mutex<()>, + + #[doc(hidden)] + #[cfg(feature = "metrics")] + pub metrics: Arc, +} + +impl TreeInner { + pub(crate) fn create_new(config: Config) -> crate::Result { + let version = Version::new( + 0, + if config.kv_separation_opts.is_some() { + crate::TreeType::Blob + } else { + crate::TreeType::Standard + }, + ); + persist_version(&config.path, &version)?; + + Ok(Self { + id: get_next_tree_id(), + memtable_id_counter: SequenceNumberCounter::new(1), + table_id_counter: SequenceNumberCounter::default(), + blob_file_id_counter: SequenceNumberCounter::default(), + config: Arc::new(config), + version_history: Arc::new(RwLock::new(SuperVersions::new(version))), + stop_signal: StopSignal::default(), + major_compaction_lock: RwLock::default(), + flush_lock: Mutex::default(), + compaction_state: Arc::new(Mutex::new(CompactionState::default())), + + #[cfg(feature = "metrics")] + metrics: Metrics::default().into(), + }) + } + + pub fn get_next_table_id(&self) -> TableId { + self.table_id_counter.next() + } +} + +impl Drop for TreeInner { + fn drop(&mut self) { + log::debug!("Dropping TreeInner"); + + log::trace!("Sending stop signal to compactors"); + self.stop_signal.send(); + } +} diff --git a/crates/lsm-tree/src/tree/mod.rs b/crates/lsm-tree/src/tree/mod.rs new file mode 100644 index 000000000..c517131ee --- /dev/null +++ b/crates/lsm-tree/src/tree/mod.rs @@ -0,0 +1,1228 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod ingest; +pub mod inner; +pub mod sealed; + +use crate::{ + AbstractTree, Checksum, KvPair, SeqNo, SequenceNumberCounter, TableId, UserKey, UserValue, + ValueType, + compaction::{CompactionStrategy, drop_range::OwnedBounds, state::CompactionState}, + config::Config, + file::CURRENT_VERSION_FILE, + format_version::FormatVersion, + iter_guard::{IterGuard, IterGuardImpl}, + key::InternalKey, + manifest::Manifest, + memtable::Memtable, + slice::Slice, + table::Table, + value::InternalValue, + version::{SuperVersion, SuperVersions, Version, recovery::recover}, + vlog::BlobFile, +}; +use inner::{TreeId, TreeInner}; +use std::{ + ops::{Bound, RangeBounds}, + path::Path, + sync::{Arc, Mutex, RwLock}, +}; + +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; + +/// Iterator value guard +pub struct Guard(crate::Result<(UserKey, UserValue)>); + +impl IterGuard for Guard { + fn into_inner_if( + self, + pred: impl Fn(&UserKey) -> bool, + ) -> crate::Result<(UserKey, Option)> { + let (k, v) = self.0?; + + if pred(&k) { + Ok((k, Some(v))) + } else { + Ok((k, None)) + } + } + + fn key(self) -> crate::Result { + self.0.map(|(k, _)| k) + } + + fn size(self) -> crate::Result { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + self.into_inner().map(|(_, v)| v.len() as u32) + } + + fn into_inner(self) -> crate::Result<(UserKey, UserValue)> { + self.0 + } +} + +fn ignore_tombstone_value(item: InternalValue) -> Option { + if item.is_tombstone() { + None + } else { + Some(item) + } +} + +/// A log-structured merge tree (LSM-tree/LSMT) +#[derive(Clone)] +pub struct Tree(#[doc(hidden)] pub Arc); + +impl std::ops::Deref for Tree { + type Target = TreeInner; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AbstractTree for Tree { + fn table_file_cache_size(&self) -> usize { + self.config + .descriptor_table + .as_ref() + .map_or(0, |dt| dt.len()) + } + + fn get_version_history_lock( + &self, + ) -> std::sync::RwLockWriteGuard<'_, crate::version::SuperVersions> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history.write().expect("lock is poisoned") + } + + fn next_table_id(&self) -> TableId { + self.0.table_id_counter.get() + } + + fn id(&self) -> TreeId { + self.id + } + + fn blob_file_count(&self) -> usize { + 0 + } + + fn print_trace(&self, key: &[u8]) -> crate::Result<()> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let super_version = self + .version_history + .read() + .expect("lock is poisoned") + .latest_version(); + + let key = Slice::from(key); + + for kv in super_version + .active_memtable + .range(InternalKey::new(key.clone(), SeqNo::MAX, ValueType::Value)..) + { + log::info!("[Active] {kv:?}"); + } + + for mt in super_version.sealed_memtables.iter().rev() { + for kv in mt.range(InternalKey::new(key.clone(), SeqNo::MAX, ValueType::Value)..) { + log::info!("[Sealed #{}] {kv:?}", mt.id()); + } + } + + for table in super_version + .version + .iter_levels() + .flat_map(|lvl| lvl.iter()) + .filter_map(|run| run.get_for_key(&key)) + { + for kv in table.range(..) { + let kv = kv?; + + if kv.key.user_key != key { + break; + } + + log::info!("[Table #{}] {kv:?}", table.id()); + } + } + + Ok(()) + } + + fn get_internal_entry(&self, key: &[u8], seqno: SeqNo) -> crate::Result> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let super_version = self + .version_history + .read() + .expect("lock is poisoned") + .get_version_arc_for_snapshot(seqno); + + Self::get_internal_entry_from_version(&super_version, key, seqno) + } + + fn current_version(&self) -> Version { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history + .read() + .expect("poisoned") + .latest_version() + .version + } + + fn get_flush_lock(&self) -> std::sync::MutexGuard<'_, ()> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.flush_lock.lock().expect("lock is poisoned") + } + + #[cfg(feature = "metrics")] + fn metrics(&self) -> &Arc { + &self.0.metrics + } + + fn version_free_list_len(&self) -> usize { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history + .read() + .expect("lock is poisoned") + .free_list_len() + } + + fn prefix>( + &self, + prefix: K, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static> { + Box::new( + self.create_prefix(&prefix, seqno, index) + .map(|kv| IterGuardImpl::Standard(Guard(kv))), + ) + } + + fn range, R: RangeBounds>( + &self, + range: R, + seqno: SeqNo, + index: Option<(Arc, SeqNo)>, + ) -> Box + Send + 'static> { + Box::new( + self.create_range(&range, seqno, index) + .map(|kv| IterGuardImpl::Standard(Guard(kv))), + ) + } + + /// Returns the number of tombstones in the tree. + fn tombstone_count(&self) -> u64 { + self.current_version() + .iter_tables() + .map(Table::tombstone_count) + .sum() + } + + /// Returns the number of weak tombstones (single deletes) in the tree. + fn weak_tombstone_count(&self) -> u64 { + self.current_version() + .iter_tables() + .map(Table::weak_tombstone_count) + .sum() + } + + /// Returns the number of value entries that become reclaimable once weak tombstones can be GC'd. + fn weak_tombstone_reclaimable_count(&self) -> u64 { + self.current_version() + .iter_tables() + .map(Table::weak_tombstone_reclaimable) + .sum() + } + + fn drop_range, R: RangeBounds>(&self, range: R) -> crate::Result<()> { + let (bounds, is_empty) = Self::range_bounds_to_owned_bounds(&range); + + if is_empty { + return Ok(()); + } + + let strategy = Arc::new(crate::compaction::drop_range::Strategy::new(bounds)); + + // IMPORTANT: Write lock so we can be the only compaction going on + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let _lock = self + .0 + .major_compaction_lock + .write() + .expect("lock is poisoned"); + + log::info!("Starting drop_range compaction"); + self.inner_compact(strategy, 0) + } + + fn clear(&self) -> crate::Result<()> { + let config = self.tree_config(); + let mut versions = self.get_version_history_lock(); + + versions.upgrade_version( + &config.path, + |v| { + let mut copy = v.clone(); + copy.active_memtable = Arc::new(Memtable::new(self.memtable_id_counter.next())); + copy.sealed_memtables = Arc::default(); + copy.version = Version::new(v.version.id() + 1, self.tree_type()); + Ok(copy) + }, + &config.seqno, + &config.visible_seqno, + ) + } + + #[doc(hidden)] + fn major_compact(&self, target_size: u64, seqno_threshold: SeqNo) -> crate::Result<()> { + let strategy = Arc::new(crate::compaction::major::Strategy::new(target_size)); + + // IMPORTANT: Write lock so we can be the only compaction going on + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let _lock = self + .0 + .major_compaction_lock + .write() + .expect("lock is poisoned"); + + log::info!("Starting major compaction"); + self.inner_compact(strategy, seqno_threshold) + } + + fn l0_run_count(&self) -> usize { + self.current_version() + .level(0) + .map(|x| x.run_count()) + .unwrap_or_default() + } + + fn size_of>(&self, key: K, seqno: SeqNo) -> crate::Result> { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + Ok(self.get(key, seqno)?.map(|x| x.len() as u32)) + } + + fn filter_size(&self) -> u64 { + self.current_version() + .iter_tables() + .map(Table::filter_size) + .map(u64::from) + .sum() + } + + fn pinned_filter_size(&self) -> usize { + self.current_version() + .iter_tables() + .map(Table::pinned_filter_size) + .sum() + } + + fn pinned_block_index_size(&self) -> usize { + self.current_version() + .iter_tables() + .map(Table::pinned_block_index_size) + .sum() + } + + fn sealed_memtable_count(&self) -> usize { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history + .read() + .expect("lock is poisoned") + .latest_version() + .sealed_memtables + .len() + } + + fn flush_to_tables( + &self, + stream: impl Iterator>, + ) -> crate::Result, Option>)>> { + use crate::{file::TABLES_FOLDER, table::multi_writer::MultiWriter}; + use std::time::Instant; + + let start = Instant::now(); + + let folder = self.config.path.join(TABLES_FOLDER); + + let data_block_size = self.config.data_block_size_policy.get(0); + + let data_block_restart_interval = self.config.data_block_restart_interval_policy.get(0); + let index_block_restart_interval = self.config.index_block_restart_interval_policy.get(0); + + let data_block_compression = self.config.data_block_compression_policy.get(0); + let index_block_compression = self.config.index_block_compression_policy.get(0); + + let data_block_hash_ratio = self.config.data_block_hash_ratio_policy.get(0); + + let index_partitioning = self.config.index_block_partitioning_policy.get(0); + let filter_partitioning = self.config.filter_block_partitioning_policy.get(0); + + log::debug!( + "Flushing memtable(s) to {}, data_block_restart_interval={data_block_restart_interval}, index_block_restart_interval={index_block_restart_interval}, data_block_size={data_block_size}, data_block_compression={data_block_compression:?}, index_block_compression={index_block_compression:?}", + folder.display(), + ); + + let mut table_writer = MultiWriter::new( + folder.clone(), + self.table_id_counter.clone(), + 64 * 1_024 * 1_024, + 0, + )? + .use_data_block_restart_interval(data_block_restart_interval) + .use_index_block_restart_interval(index_block_restart_interval) + .use_data_block_compression(data_block_compression) + .use_index_block_compression(index_block_compression) + .use_data_block_size(data_block_size) + .use_data_block_hash_ratio(data_block_hash_ratio) + .use_bloom_policy({ + use crate::config::FilterPolicyEntry::{Bloom, None}; + use crate::table::filter::BloomConstructionPolicy; + + match self.config.filter_policy.get(0) { + Bloom(policy) => policy, + None => BloomConstructionPolicy::BitsPerKey(0.0), + } + }); + + if index_partitioning { + table_writer = table_writer.use_partitioned_index(); + } + if filter_partitioning { + table_writer = table_writer.use_partitioned_filter(); + } + + for item in stream { + table_writer.write(item?)?; + } + + let result = table_writer.finish()?; + + log::debug!("Flushed memtable(s) in {:?}", start.elapsed()); + + let pin_filter = self.config.filter_block_pinning_policy.get(0); + let pin_index = self.config.index_block_pinning_policy.get(0); + + // Load tables + let tables = result + .into_iter() + .map(|(table_id, checksum)| -> crate::Result
{ + Table::recover( + folder.join(table_id.to_string()), + checksum, + 0, + self.id, + self.config.cache.clone(), + self.config.descriptor_table.clone(), + pin_filter, + pin_index, + #[cfg(feature = "metrics")] + self.metrics.clone(), + ) + }) + .collect::>>()?; + + Ok(Some((tables, None))) + } + + #[expect(clippy::significant_drop_tightening)] + fn register_tables( + &self, + tables: &[Table], + blob_files: Option<&[BlobFile]>, + frag_map: Option, + sealed_memtables_to_delete: &[crate::tree::inner::MemtableId], + gc_watermark: SeqNo, + ) -> crate::Result<()> { + log::trace!( + "Registering {} tables, {} blob files", + tables.len(), + blob_files.map(<[BlobFile]>::len).unwrap_or_default(), + ); + + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut _compaction_state = self.compaction_state.lock().expect("lock is poisoned"); + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_lock = self.version_history.write().expect("lock is poisoned"); + + // NOTE: Check for race condition + // Fixes: https://github.com/fjall-rs/fjall/issues/287#issuecomment-4938188362 + if sealed_memtables_to_delete + .iter() + .any(|id| !version_lock.latest_version().sealed_memtables.contains(*id)) + { + log::debug!( + "Not registering tables because flush task processed some sealed memtables which do not exist (anymore)" + ); + return Ok(()); + } + + version_lock.upgrade_version( + &self.config.path, + |current| { + let mut copy = current.clone(); + + copy.version = copy.version.with_new_l0_run( + tables, + blob_files, + frag_map.filter(|x| !x.is_empty()), + ); + + for &table_id in sealed_memtables_to_delete { + log::trace!("releasing sealed memtable #{table_id}"); + copy.sealed_memtables = Arc::new(copy.sealed_memtables.remove(table_id)); + } + + Ok(copy) + }, + &self.config.seqno, + &self.config.visible_seqno, + )?; + + if let Err(e) = version_lock.maintenance(&self.config.path, gc_watermark) { + log::warn!("Version GC failed: {e:?}"); + } + + Ok(()) + } + + fn clear_active_memtable(&self) { + use crate::tree::sealed::SealedMemtables; + + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_history_lock = self.version_history.write().expect("lock is poisoned"); + let super_version = version_history_lock.latest_version(); + + if super_version.active_memtable.is_empty() { + return; + } + + let mut copy = version_history_lock.latest_version(); + copy.active_memtable = Arc::new(Memtable::new(self.memtable_id_counter.next())); + copy.sealed_memtables = Arc::new(SealedMemtables::default()); + + // Clear active is only used for recovery where snapshots do not exist yet + copy.seqno = super_version.seqno; + + version_history_lock.replace_latest_version(copy); + + log::trace!("cleared active memtable"); + } + + fn compact( + &self, + strategy: Arc, + seqno_threshold: SeqNo, + ) -> crate::Result<()> { + // NOTE: Read lock major compaction lock + // That way, if a major compaction is running, we cannot proceed + // But in general, parallel (non-major) compactions can occur + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let _lock = self + .0 + .major_compaction_lock + .read() + .expect("lock is poisoned"); + + self.inner_compact(strategy, seqno_threshold) + } + + fn get_next_table_id(&self) -> TableId { + self.0.get_next_table_id() + } + + fn tree_config(&self) -> &Config { + &self.config + } + + fn active_memtable(&self) -> Arc { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history + .read() + .expect("lock is poisoned") + .latest_version() + .active_memtable + } + + #[expect(clippy::significant_drop_tightening)] + fn rotate_memtable(&self) -> Option> { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let mut version_history_lock = self.version_history.write().expect("lock is poisoned"); + let super_version = version_history_lock.latest_version(); + + if super_version.active_memtable.is_empty() { + return None; + } + + let yanked_memtable = super_version.active_memtable; + + let mut copy = version_history_lock.latest_version(); + copy.active_memtable = Arc::new(Memtable::new(self.memtable_id_counter.next())); + copy.sealed_memtables = + Arc::new(super_version.sealed_memtables.add(yanked_memtable.clone())); + + // Rotate does not modify the memtable so it cannot break snapshots + copy.seqno = super_version.seqno; + + version_history_lock.replace_latest_version(copy); + + log::trace!( + "rotate: added memtable id={} to sealed memtables", + yanked_memtable.id, + ); + + Some(yanked_memtable) + } + + fn table_count(&self) -> usize { + self.current_version().table_count() + } + + fn level_table_count(&self, idx: usize) -> Option { + self.current_version().level(idx).map(|x| x.table_count()) + } + + fn approximate_len(&self) -> usize { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let super_version = self + .version_history + .read() + .expect("lock is poisoned") + .latest_version(); + + let tables_item_count = self + .current_version() + .iter_tables() + .map(|x| x.metadata.item_count) + .sum::(); + + let memtable_count = super_version.active_memtable.len() as u64; + let sealed_count = super_version + .sealed_memtables + .iter() + .map(|mt| mt.len()) + .sum::() as u64; + + #[expect(clippy::expect_used, reason = "result should fit into usize")] + (memtable_count + sealed_count + tables_item_count) + .try_into() + .expect("approximate_len too large for usize") + } + + fn disk_space(&self) -> u64 { + self.current_version() + .iter_levels() + .map(super::version::Level::size) + .sum() + } + + fn get_highest_memtable_seqno(&self) -> Option { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let version = self + .version_history + .read() + .expect("lock is poisoned") + .latest_version(); + + let active = version.active_memtable.get_highest_seqno(); + + let sealed = version + .sealed_memtables + .iter() + .map(|mt| mt.get_highest_seqno()) + .max() + .flatten(); + + active.max(sealed) + } + + fn get_highest_persisted_seqno(&self) -> Option { + self.current_version() + .iter_tables() + .map(Table::get_highest_seqno) + .max() + } + + fn get>(&self, key: K, seqno: SeqNo) -> crate::Result> { + let key = key.as_ref(); + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let super_version = self + .version_history + .read() + .expect("lock is poisoned") + .get_version_arc_for_snapshot(seqno); + + if let Some(entry) = super_version.active_memtable.get(key, seqno) { + return Ok(ignore_tombstone_value(entry).map(|entry| entry.value)); + } + + if let Some(entry) = + Self::get_internal_entry_from_sealed_memtables(&super_version, key, seqno) + { + return Ok(ignore_tombstone_value(entry).map(|entry| entry.value)); + } + + let key_hash = crate::table::filter::standard_bloom::Builder::get_hash(key); + for table in super_version + .version + .iter_levels() + .flat_map(|level| level.iter()) + .filter_map(|run| run.get_for_key(key)) + { + if let Some(item) = table.get_value(key, seqno, key_hash)? { + return Ok((!item.value_type.is_tombstone()).then_some(item.value)); + } + } + + Ok(None) + } + + fn insert, V: Into>( + &self, + key: K, + value: V, + seqno: SeqNo, + ) -> (u64, u64) { + let value = InternalValue::from_components(key, value, seqno, ValueType::Value); + self.append_entry(value) + } + + fn remove>(&self, key: K, seqno: SeqNo) -> (u64, u64) { + let value = InternalValue::new_tombstone(key, seqno); + self.append_entry(value) + } + + fn remove_weak>(&self, key: K, seqno: SeqNo) -> (u64, u64) { + let value = InternalValue::new_weak_tombstone(key, seqno); + self.append_entry(value) + } +} + +impl Tree { + #[doc(hidden)] + pub fn create_internal_range<'a, K: AsRef<[u8]> + 'a, R: RangeBounds + 'a>( + version: SuperVersion, + range: &'a R, + seqno: SeqNo, + ephemeral: Option<(Arc, SeqNo)>, + ) -> impl DoubleEndedIterator> + 'static + use { + use crate::range::{IterState, TreeIter}; + use std::ops::Bound::{self, Excluded, Included, Unbounded}; + + let lo: Bound = match range.start_bound() { + Included(x) => Included(x.as_ref().into()), + Excluded(x) => Excluded(x.as_ref().into()), + Unbounded => Unbounded, + }; + + let hi: Bound = match range.end_bound() { + Included(x) => Included(x.as_ref().into()), + Excluded(x) => Excluded(x.as_ref().into()), + Unbounded => Unbounded, + }; + + let bounds: (Bound, Bound) = (lo, hi); + + let iter_state = { IterState { version, ephemeral } }; + + TreeIter::create_range(iter_state, bounds, seqno) + } + + pub(crate) fn get_internal_entry_from_version( + super_version: &SuperVersion, + key: &[u8], + seqno: SeqNo, + ) -> crate::Result> { + if let Some(entry) = super_version.active_memtable.get(key, seqno) { + return Ok(ignore_tombstone_value(entry)); + } + + // Now look in sealed memtables + if let Some(entry) = + Self::get_internal_entry_from_sealed_memtables(super_version, key, seqno) + { + return Ok(ignore_tombstone_value(entry)); + } + + // Now look in tables... this may involve disk I/O + Self::get_internal_entry_from_tables(&super_version.version, key, seqno) + } + + fn get_internal_entry_from_tables( + version: &Version, + key: &[u8], + seqno: SeqNo, + ) -> crate::Result> { + // NOTE: Create key hash for hash sharing + // https://fjall-rs.github.io/post/bloom-filter-hash-sharing/ + let key_hash = crate::table::filter::standard_bloom::Builder::get_hash(key); + + for table in version + .iter_levels() + .flat_map(|lvl| lvl.iter()) + .filter_map(|run| run.get_for_key(key)) + { + if let Some(item) = table.get(key, seqno, key_hash)? { + return Ok(ignore_tombstone_value(item)); + } + } + + Ok(None) + } + + fn get_internal_entry_from_sealed_memtables( + super_version: &SuperVersion, + key: &[u8], + seqno: SeqNo, + ) -> Option { + for mt in super_version.sealed_memtables.iter().rev() { + if let Some(entry) = mt.get(key, seqno) { + return Some(entry); + } + } + + None + } + + pub(crate) fn get_version_for_snapshot(&self, seqno: SeqNo) -> SuperVersion { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history + .read() + .expect("lock is poisoned") + .get_version_for_snapshot(seqno) + } + + /// Normalizes a user-provided range into owned `Bound` values. + /// + /// Returns a tuple containing: + /// - the `OwnedBounds` that mirror the original bounds semantics (including + /// inclusive/exclusive markers and unbounded endpoints), and + /// - a `bool` flag indicating whether the normalized range is logically + /// empty (e.g., when the lower bound is greater than the upper bound). + /// + /// Callers can use the flag to detect empty ranges and skip further work + /// while still having access to the normalized bounds for non-empty cases. + fn range_bounds_to_owned_bounds, R: RangeBounds>( + range: &R, + ) -> (OwnedBounds, bool) { + use Bound::{Excluded, Included, Unbounded}; + + let start = match range.start_bound() { + Included(key) => Included(Slice::from(key.as_ref())), + Excluded(key) => Excluded(Slice::from(key.as_ref())), + Unbounded => Unbounded, + }; + + let end = match range.end_bound() { + Included(key) => Included(Slice::from(key.as_ref())), + Excluded(key) => Excluded(Slice::from(key.as_ref())), + Unbounded => Unbounded, + }; + + let is_empty = + if let (Included(lo) | Excluded(lo), Included(hi) | Excluded(hi)) = (&start, &end) { + lo.as_ref() > hi.as_ref() + } else { + false + }; + + (OwnedBounds { start, end }, is_empty) + } + + /// Opens an LSM-tree in the given directory. + /// + /// Will recover previous state if the folder was previously + /// occupied by an LSM-tree, including the previous configuration. + /// If not, a new tree will be initialized with the given config. + /// + /// After recovering a previous state, use [`Tree::set_active_memtable`] + /// to fill the memtable with data from a write-ahead log for full durability. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + pub(crate) fn open(config: Config) -> crate::Result { + log::debug!("Opening LSM-tree at {}", config.path.display()); + + // Check for old version + if config.path.join("version").try_exists()? { + log::error!( + "It looks like you are trying to open a V1 database - the database needs a manual migration, however a migration tool is not provided, as V1 is extremely outdated." + ); + return Err(crate::Error::InvalidVersion(FormatVersion::V1.into())); + } + + let tree = if config.path.join(CURRENT_VERSION_FILE).try_exists()? { + Self::recover(config) + } else { + Self::create_new(config) + }?; + + Ok(tree) + } + + /// Returns `true` if there are some tables that are being compacted. + #[doc(hidden)] + #[must_use] + pub fn is_compacting(&self) -> bool { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + !self + .compaction_state + .lock() + .expect("lock is poisoned") + .hidden_set() + .is_empty() + } + + fn inner_compact( + &self, + strategy: Arc, + mvcc_gc_watermark: SeqNo, + ) -> crate::Result<()> { + use crate::compaction::worker::{Options, do_compaction}; + + let mut opts = Options::from_tree(self, strategy); + opts.mvcc_gc_watermark = mvcc_gc_watermark; + + do_compaction(&opts)?; + + log::debug!("Compaction run over"); + + Ok(()) + } + + #[doc(hidden)] + #[must_use] + pub fn create_iter( + &self, + seqno: SeqNo, + ephemeral: Option<(Arc, SeqNo)>, + ) -> impl DoubleEndedIterator> + 'static + use<> { + self.create_range::(&.., seqno, ephemeral) + } + + #[doc(hidden)] + pub fn create_range<'a, K: AsRef<[u8]> + 'a, R: RangeBounds + 'a>( + &self, + range: &'a R, + seqno: SeqNo, + ephemeral: Option<(Arc, SeqNo)>, + ) -> impl DoubleEndedIterator> + 'static + use { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + let super_version = self + .version_history + .read() + .expect("lock is poisoned") + .get_version_for_snapshot(seqno); + + Self::create_internal_range(super_version, range, seqno, ephemeral).map(|item| match item { + Ok(kv) => Ok((kv.key.user_key, kv.value)), + Err(e) => Err(e), + }) + } + + #[doc(hidden)] + pub fn create_prefix<'a, K: AsRef<[u8]> + 'a>( + &self, + prefix: K, + seqno: SeqNo, + ephemeral: Option<(Arc, SeqNo)>, + ) -> impl DoubleEndedIterator> + 'static + use { + use crate::range::prefix_to_range; + + let range = prefix_to_range(prefix.as_ref()); + self.create_range(&range, seqno, ephemeral) + } + + /// Adds an item to the active memtable. + /// + /// Returns the added item's size and new size of the memtable. + #[doc(hidden)] + #[must_use] + pub fn append_entry(&self, value: InternalValue) -> (u64, u64) { + #[expect(clippy::expect_used, reason = "lock is expected to not be poisoned")] + self.version_history + .read() + .expect("lock is poisoned") + .latest_version() + .active_memtable + .insert(value) + } + + /// Recovers previous state, by loading the level manifest, tables and blob files. + /// + /// # Errors + /// + /// Returns error, if an IO error occurred. + fn recover(mut config: Config) -> crate::Result { + use crate::stop_signal::StopSignal; + use inner::get_next_tree_id; + + log::info!("Recovering LSM-tree at {}", config.path.display()); + + let tree_id = get_next_tree_id(); + + #[cfg(feature = "metrics")] + let metrics = Arc::new(Metrics::default()); + + let version = Self::recover_levels( + &config.path, + tree_id, + &config, + #[cfg(feature = "metrics")] + &metrics, + )?; + + { + let manifest_path = config.path.join(format!("v{}", version.id())); + let reader = sfa::Reader::new(&manifest_path)?; + let manifest = Manifest::decode_from(&manifest_path, &reader)?; + + if manifest.version != FormatVersion::V4 { + return Err(crate::Error::InvalidVersion(manifest.version.into())); + } + + let requested_tree_type = match config.kv_separation_opts { + Some(_) => crate::TreeType::Blob, + None => crate::TreeType::Standard, + }; + + if version.tree_type() != requested_tree_type { + log::error!( + "Tried to open a {requested_tree_type:?}Tree, but the existing tree is of type {:?}Tree. This indicates a misconfiguration or corruption.", + version.tree_type(), + ); + return Err(crate::Error::Unrecoverable); + } + + // IMPORTANT: Restore persisted config + config.level_count = manifest.level_count; + } + + let highest_table_id = version + .iter_tables() + .map(Table::id) + .max() + .unwrap_or_default(); + + let inner = TreeInner { + id: tree_id, + memtable_id_counter: SequenceNumberCounter::new(1), + table_id_counter: SequenceNumberCounter::new(highest_table_id + 1), + blob_file_id_counter: SequenceNumberCounter::default(), + version_history: Arc::new(RwLock::new(SuperVersions::new(version))), + stop_signal: StopSignal::default(), + config: Arc::new(config), + major_compaction_lock: RwLock::default(), + flush_lock: Mutex::default(), + compaction_state: Arc::new(Mutex::new(CompactionState::default())), + + #[cfg(feature = "metrics")] + metrics, + }; + + Ok(Self(Arc::new(inner))) + } + + /// Creates a new LSM-tree in a directory. + fn create_new(config: Config) -> crate::Result { + use crate::file::{TABLES_FOLDER, fsync_directory}; + use std::fs::create_dir_all; + + let path = config.path.clone(); + log::trace!("Creating LSM-tree at {}", path.display()); + + create_dir_all(&path)?; + + let table_folder_path = path.join(TABLES_FOLDER); + create_dir_all(&table_folder_path)?; + + // IMPORTANT: fsync folders on Unix + fsync_directory(&table_folder_path)?; + fsync_directory(&path)?; + + let inner = TreeInner::create_new(config)?; + Ok(Self(Arc::new(inner))) + } + + /// Recovers the level manifest, loading all tables from disk. + fn recover_levels>( + tree_path: P, + tree_id: TreeId, + config: &Config, + #[cfg(feature = "metrics")] metrics: &Arc, + ) -> crate::Result { + use crate::{TableId, file::TABLES_FOLDER, file::fsync_directory}; + + let tree_path = tree_path.as_ref(); + + let recovery = recover(tree_path)?; + + let table_map = { + let mut result: crate::HashMap = + crate::HashMap::default(); + + for (level_idx, table_ids) in recovery.table_ids.iter().enumerate() { + for run in table_ids { + for table in run { + #[expect( + clippy::expect_used, + reason = "there are always less than 256 levels" + )] + result.insert( + table.id, + ( + level_idx + .try_into() + .expect("there are less than 256 levels"), + table.checksum, + table.global_seqno, + ), + ); + } + } + } + + result + }; + + let cnt = table_map.len(); + + log::debug!("Recovering {cnt} tables from {}", tree_path.display()); + + let progress_mod = match cnt { + _ if cnt <= 20 => 1, + _ if cnt <= 100 => 10, + _ => 100, + }; + + let mut tables = vec![]; + + let table_base_folder = tree_path.join(TABLES_FOLDER); + + if !table_base_folder.try_exists()? { + std::fs::create_dir_all(&table_base_folder)?; + fsync_directory(&table_base_folder)?; + } + + let mut orphaned_tables = vec![]; + + for (idx, dirent) in std::fs::read_dir(&table_base_folder)?.enumerate() { + let dirent = dirent?; + let file_name = dirent.file_name(); + + // https://en.wikipedia.org/wiki/.DS_Store + if file_name == ".DS_Store" { + continue; + } + + // https://en.wikipedia.org/wiki/AppleSingle_and_AppleDouble_formats + if file_name.to_string_lossy().starts_with("._") { + continue; + } + + let table_file_name = file_name.to_str().ok_or_else(|| { + log::error!("invalid table file name {}", file_name.display()); + crate::Error::Unrecoverable + })?; + + let table_file_path = dirent.path(); + assert!(!table_file_path.is_dir()); + + let table_id = table_file_name.parse::().map_err(|e| { + log::error!("invalid table file name {table_file_name:?}: {e:?}"); + crate::Error::Unrecoverable + })?; + + if let Some(&(level_idx, checksum, global_seqno)) = table_map.get(&table_id) { + let pin_filter = config.filter_block_pinning_policy.get(level_idx.into()); + let pin_index = config.index_block_pinning_policy.get(level_idx.into()); + + let table = Table::recover( + table_file_path, + checksum, + global_seqno, + tree_id, + config.cache.clone(), + config.descriptor_table.clone(), + pin_filter, + pin_index, + #[cfg(feature = "metrics")] + metrics.clone(), + )?; + + tables.push(table); + + if idx % progress_mod == 0 { + log::debug!("Recovered {idx}/{cnt} tables"); + } + } else { + orphaned_tables.push(table_file_path); + } + } + + if tables.len() < cnt { + log::error!( + "Recovered less tables than expected: {:?}", + table_map.keys(), + ); + return Err(crate::Error::Unrecoverable); + } + + log::debug!("Successfully recovered {} tables", tables.len()); + + let (blob_files, orphaned_blob_files) = crate::vlog::recover_blob_files( + &tree_path.join(crate::file::BLOBS_FOLDER), + &recovery.blob_file_ids, + tree_id, + config.descriptor_table.as_ref(), + )?; + + let version = Version::from_recovery(recovery, &tables, &blob_files)?; + + // NOTE: Cleanup old versions + // But only after we definitely recovered the latest version + Self::cleanup_orphaned_version(tree_path, version.id())?; + + for table_path in orphaned_tables { + log::debug!("Deleting orphaned table {}", table_path.display()); + std::fs::remove_file(&table_path)?; + } + + for blob_file_path in orphaned_blob_files { + log::debug!("Deleting orphaned blob file {}", blob_file_path.display()); + std::fs::remove_file(&blob_file_path)?; + } + + Ok(version) + } + + fn cleanup_orphaned_version( + path: &Path, + latest_version_id: crate::version::VersionId, + ) -> crate::Result<()> { + let version_str = format!("v{latest_version_id}"); + + for file in std::fs::read_dir(path)? { + let dirent = file?; + + if dirent.file_type()?.is_dir() { + continue; + } + + let name = dirent.file_name(); + + if name.to_string_lossy().starts_with('v') && *name != *version_str { + log::trace!("Cleanup orphaned version {}", name.display()); + std::fs::remove_file(dirent.path())?; + } + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/tree/sealed.rs b/crates/lsm-tree/src/tree/sealed.rs new file mode 100644 index 000000000..4408e8350 --- /dev/null +++ b/crates/lsm-tree/src/tree/sealed.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{Memtable, tree::inner::MemtableId}; +use std::sync::Arc; + +/// Stores references to all sealed memtables +/// +/// Memtable IDs are monotonically increasing, so we don't really +/// need a search tree; also there are only a handful of them at most. +#[derive(Clone, Default)] +pub struct SealedMemtables(Vec>); + +impl SealedMemtables { + /// Copy-and-writes a new list with additional Memtable. + pub fn add(&self, memtable: Arc) -> Self { + let mut copy = self.clone(); + copy.0.push(memtable); + copy + } + + /// Copy-and-writes a new list with the specified Memtable removed. + pub fn remove(&self, id_to_remove: MemtableId) -> Self { + let mut copy = self.clone(); + copy.0.retain(|mt| mt.id != id_to_remove); + copy + } + + pub fn iter(&self) -> impl DoubleEndedIterator> { + self.0.iter() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn contains(&self, id: MemtableId) -> bool { + self.0.iter().any(|t| t.id() == id) + } +} diff --git a/crates/lsm-tree/src/util.rs b/crates/lsm-tree/src/util.rs new file mode 100644 index 000000000..444426286 --- /dev/null +++ b/crates/lsm-tree/src/util.rs @@ -0,0 +1,173 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::range::prefix_upper_range; +use crate::UserKey; +use std::ops::RangeBounds; + +pub use crate::range::prefix_to_range; + +/// Helper function to create a prefixed range. +/// +/// Made for phil. +/// +/// # Panics +/// +/// Panics if the prefix is empty. +pub fn prefixed_range, K: AsRef<[u8]>, R: RangeBounds>( + prefix: P, + range: R, +) -> impl RangeBounds { + use std::ops::Bound::{Excluded, Included, Unbounded}; + + let prefix = prefix.as_ref(); + + if prefix.is_empty() { + return (Unbounded, Unbounded); + } + + match (range.start_bound(), range.end_bound()) { + (Unbounded, Unbounded) => prefix_to_range(prefix), + (lower, Unbounded) => { + let lower = lower.map(|k| UserKey::fused(prefix, k.as_ref())); + let upper = prefix_upper_range(prefix); + (lower, upper) + } + (Unbounded, upper) => { + let upper = match upper { + Included(k) => Included(UserKey::fused(prefix, k.as_ref())), + Excluded(k) => Excluded(UserKey::fused(prefix, k.as_ref())), + Unbounded => unreachable!(), + }; + + (Included(prefix.into()), upper) + } + (lower, upper) => { + let lower = match lower { + Included(k) => Included(UserKey::fused(prefix, k.as_ref())), + Excluded(k) => Excluded(UserKey::fused(prefix, k.as_ref())), + Unbounded => unreachable!(), + }; + + let upper = match upper { + Included(k) => Included(UserKey::fused(prefix, k.as_ref())), + Excluded(k) => Excluded(UserKey::fused(prefix, k.as_ref())), + Unbounded => unreachable!(), + }; + + (lower, upper) + } + } +} + +#[cfg(test)] +mod tests { + use super::prefixed_range; + use crate::UserKey; + use std::ops::Bound::{Excluded, Included}; + use std::ops::RangeBounds; + use test_log::test; + + #[test] + fn prefixed_range_1() { + let prefix = "abc"; + let min = 5u8.to_be_bytes(); + let max = 9u8.to_be_bytes(); + + let range = prefixed_range(prefix, min..=max); + + assert_eq!( + range.start_bound(), + Included(&UserKey::new(&[b'a', b'b', b'c', 5])) + ); + assert_eq!( + range.end_bound(), + Included(&UserKey::new(&[b'a', b'b', b'c', 9])) + ); + } + + #[test] + fn prefixed_range_2() { + let prefix = "abc"; + let min = 5u8.to_be_bytes(); + let max = 9u8.to_be_bytes(); + + let range = prefixed_range(prefix, min..max); + + assert_eq!( + range.start_bound(), + Included(&UserKey::new(&[b'a', b'b', b'c', 5])) + ); + assert_eq!( + range.end_bound(), + Excluded(&UserKey::new(&[b'a', b'b', b'c', 9])) + ); + } + + #[test] + fn prefixed_range_3() { + let prefix = "abc"; + let min = 5u8.to_be_bytes(); + + let range = prefixed_range(prefix, min..); + + assert_eq!( + range.start_bound(), + Included(&UserKey::new(&[b'a', b'b', b'c', 5])) + ); + assert_eq!(range.end_bound(), Excluded(&UserKey::new(b"abd"))); + } + + #[test] + fn prefixed_range_4() { + let prefix = "abc"; + let max = 9u8.to_be_bytes(); + + let range = prefixed_range(prefix, ..max); + + assert_eq!(range.start_bound(), Included(&UserKey::new(b"abc"))); + assert_eq!( + range.end_bound(), + Excluded(&UserKey::new(&[b'a', b'b', b'c', 9])) + ); + } + + #[test] + fn prefixed_range_5() { + let prefix = "abc"; + let max = u8::MAX.to_be_bytes(); + + let range = prefixed_range(prefix, ..=max); + + assert_eq!(range.start_bound(), Included(&UserKey::new(b"abc"))); + assert_eq!( + range.end_bound(), + Included(&UserKey::new(&[b'a', b'b', b'c', u8::MAX])) + ); + } + + #[test] + fn prefixed_range_6() { + let prefix = "abc"; + let max = u8::MAX.to_be_bytes(); + + let range = prefixed_range(prefix, ..max); + + assert_eq!(range.start_bound(), Included(&UserKey::new(b"abc"))); + assert_eq!( + range.end_bound(), + Excluded(&UserKey::new(&[b'a', b'b', b'c', u8::MAX])) + ); + } + + #[test] + fn prefixed_range_7() { + let prefix = "abc"; + + let range = prefixed_range::<_, &[u8], _>(prefix, ..); + + assert_eq!(range.start_bound(), Included(&UserKey::new(b"abc"))); + assert_eq!(range.end_bound(), Excluded(&UserKey::new(b"abd"))); + } +} diff --git a/crates/lsm-tree/src/value.rs b/crates/lsm-tree/src/value.rs new file mode 100644 index 000000000..e872160e4 --- /dev/null +++ b/crates/lsm-tree/src/value.rs @@ -0,0 +1,143 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{key::InternalKey, Slice, ValueType}; + +/// User defined key +pub type UserKey = Slice; + +/// User defined data (blob of bytes) +pub type UserValue = Slice; + +/// Sequence number - a monotonically increasing counter +/// +/// Values with the same seqno are part of the same batch. +/// +/// A value with a higher sequence number shadows an item with the +/// same key and lower sequence number. +/// This enables MVCC. +/// +/// Stale items are lazily garbage-collected during compaction. +pub type SeqNo = u64; + +pub struct PointReadValue { + pub value_type: ValueType, + pub value: UserValue, +} + +/// Internal representation of KV pairs +#[derive(Clone)] +pub struct InternalValue { + /// Internal key + pub key: InternalKey, + + /// User-defined value - an arbitrary byte array + /// + /// Supports up to 2^32 bytes + pub value: UserValue, +} + +impl InternalValue { + /// Creates a new [`Value`]. + /// + /// # Panics + /// + /// Panics if the key length is empty or greater than 2^16, or the value length is greater than 2^32. + pub fn new>(key: InternalKey, value: V) -> Self { + let value = value.into(); + + assert!(!key.user_key.is_empty(), "key may not be empty"); + assert!( + u32::try_from(value.len()).is_ok(), + "values can be 2^32 bytes in length" + ); + + Self { key, value } + } + + /// Creates a new [`Value`]. + /// + /// # Panics + /// + /// Panics if the key length is empty or greater than 2^16, or the value length is greater than 2^32. + pub fn from_components, V: Into>( + user_key: K, + value: V, + seqno: SeqNo, + value_type: ValueType, + ) -> Self { + let key = InternalKey::new(user_key, seqno, value_type); + Self::new(key, value) + } + + /// Creates a new tombstone. + /// + /// # Panics + /// + /// Panics if the key length is empty or greater than 2^16. + pub fn new_tombstone>(key: K, seqno: u64) -> Self { + let key = InternalKey::new(key, seqno, ValueType::Tombstone); + Self::new(key, vec![]) + } + + /// Creates a new weak tombstone. + /// + /// # Panics + /// + /// Panics if the key length is empty or greater than 2^16. + pub fn new_weak_tombstone>(key: K, seqno: u64) -> Self { + let key = InternalKey::new(key, seqno, ValueType::WeakTombstone); + Self::new(key, vec![]) + } + + #[doc(hidden)] + #[must_use] + pub fn is_tombstone(&self) -> bool { + self.key.is_tombstone() + } +} + +#[cfg(test)] +#[cfg_attr(coverage_nightly, coverage(off))] +impl PartialEq for InternalValue { + fn eq(&self, other: &Self) -> bool { + self.key == other.key + } +} + +#[cfg_attr(coverage_nightly, coverage(off))] +impl std::fmt::Debug for InternalValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:?} => {:?}", + self.key, + if self.value.len() >= 100 { + format!("[ ... {} bytes ]", self.value.len()) + } else { + format!("{:?}", self.value) + } + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn pik_cmp_user_key() { + let a = InternalKey::new(*b"a", 0, ValueType::Value); + let b = InternalKey::new(*b"b", 0, ValueType::Value); + assert!(a < b); + } + + #[test] + fn pik_cmp_seqno() { + let a = InternalKey::new(*b"a", 0, ValueType::Value); + let b = InternalKey::new(*b"a", 1, ValueType::Value); + assert!(a > b); + } +} diff --git a/crates/lsm-tree/src/value_type.rs b/crates/lsm-tree/src/value_type.rs new file mode 100644 index 000000000..fa4774999 --- /dev/null +++ b/crates/lsm-tree/src/value_type.rs @@ -0,0 +1,59 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +/// Value type (regular value or tombstone) +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[cfg_attr(test, derive(strum::EnumIter))] +pub enum ValueType { + /// Existing value + Value, + + /// Deleted value + Tombstone, + + /// "Weak" deletion (a.k.a. `SingleDelete` in `RocksDB`) + WeakTombstone, + + /// Value pointer + /// + /// Points to a blob in a blob file. + Indirection = 4, +} + +impl ValueType { + /// Returns `true` if the type is a tombstone marker (either normal or weak). + #[must_use] + pub fn is_tombstone(self) -> bool { + self == Self::Tombstone || self == Self::WeakTombstone + } + + pub(crate) fn is_indirection(self) -> bool { + self == Self::Indirection + } +} + +impl TryFrom for ValueType { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Value), + 1 => Ok(Self::Tombstone), + 2 => Ok(Self::WeakTombstone), + 4 => Ok(Self::Indirection), + _ => Err(()), + } + } +} + +impl From for u8 { + fn from(value: ValueType) -> Self { + match value { + ValueType::Value => 0, + ValueType::Tombstone => 1, + ValueType::WeakTombstone => 2, + ValueType::Indirection => 4, + } + } +} diff --git a/crates/lsm-tree/src/version/blob_file_list.rs b/crates/lsm-tree/src/version/blob_file_list.rs new file mode 100644 index 000000000..1c595126a --- /dev/null +++ b/crates/lsm-tree/src/version/blob_file_list.rs @@ -0,0 +1,60 @@ +use crate::{ + blob_tree::FragmentationMap, + vlog::{BlobFile, BlobFileId}, + HashMap, +}; + +// TODO: MSRV: [1.91] Replace with BTreeMap + extract_if +#[derive(Clone, Default)] +pub struct BlobFileList(HashMap); + +impl BlobFileList { + pub fn new(blob_files: HashMap) -> Self { + Self(blob_files) + } + + /// On-disk size + pub fn on_disk_size(&self) -> u64 { + self.iter().map(|bf| bf.0.meta.total_compressed_bytes).sum() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn extend>(&mut self, iter: I) { + self.0.extend(iter); + } + + pub fn contains_key(&self, key: BlobFileId) -> bool { + self.0.contains_key(&key) + } + + pub fn prune_dead(&mut self, gc_stats: &FragmentationMap) -> Vec { + self.0 + .extract_if(|_, blob_file| blob_file.is_dead(gc_stats)) + .map(|(_, v)| v) + .collect() + } + + pub fn insert(&mut self, key: BlobFileId, value: BlobFile) { + self.0.insert(key, value); + } + + pub fn remove(&mut self, key: BlobFileId) -> Option { + self.0.remove(&key) + } + + /// Visits all blob files in no particular order. + pub fn iter(&self) -> impl Iterator { + self.0.values() + } + + pub fn list_ids(&self) -> impl Iterator { + self.0.keys() + } + + pub fn get(&self, key: BlobFileId) -> Option<&BlobFile> { + self.0.get(&key) + } +} diff --git a/crates/lsm-tree/src/version/mod.rs b/crates/lsm-tree/src/version/mod.rs new file mode 100644 index 000000000..2126ff1c5 --- /dev/null +++ b/crates/lsm-tree/src/version/mod.rs @@ -0,0 +1,704 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod blob_file_list; +mod optimize; +mod persist; +pub mod recovery; +pub mod run; +mod super_version; + +pub use blob_file_list::BlobFileList; +pub use persist::persist_version; +pub use run::Run; +pub use super_version::{SuperVersion, SuperVersions}; + +use crate::blob_tree::{FragmentationEntry, FragmentationMap}; +use crate::checksum::ChecksumType; +use crate::coding::Encode; +use crate::compaction::state::hidden_set::HiddenSet; +use crate::version::recovery::Recovery; +use crate::TreeType; +use crate::{ + vlog::{BlobFile, BlobFileId}, + HashSet, KeyRange, Table, TableId, +}; +use optimize::optimize_runs; +use run::Ranged; +use std::{ops::Deref, sync::Arc}; + +pub const DEFAULT_LEVEL_COUNT: u8 = 7; + +/// Monotonically increasing ID of a version. +pub type VersionId = u64; + +impl Ranged for Table { + fn key_range(&self) -> &KeyRange { + &self.metadata.key_range + } +} + +pub struct GenericLevel { + runs: Vec>>, +} + +impl std::ops::Deref for GenericLevel { + type Target = [Arc>]; + + fn deref(&self) -> &Self::Target { + &self.runs + } +} + +impl GenericLevel { + pub fn new(runs: Vec>>) -> Self { + Self { runs } + } + + pub fn table_count(&self) -> usize { + self.iter().map(|x| x.len()).sum() + } + + pub fn run_count(&self) -> usize { + self.runs.len() + } + + pub fn is_disjoint(&self) -> bool { + self.run_count() == 1 + } + + pub fn is_empty(&self) -> bool { + self.runs.is_empty() + } + + pub fn iter(&self) -> impl DoubleEndedIterator>> { + self.runs.iter() + } +} + +#[derive(Clone)] +pub struct Level(Arc>); + +impl std::ops::Deref for Level { + type Target = GenericLevel
; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Level { + pub fn empty() -> Self { + Self::from_runs(vec![]) + } + + pub fn from_runs(runs: Vec>>) -> Self { + Self(Arc::new(GenericLevel { runs })) + } + + pub fn list_ids(&self) -> HashSet { + self.iter() + .flat_map(|run| run.iter()) + .map(Table::id) + .collect() + } + + pub fn first_run(&self) -> Option<&Arc>> { + self.runs.first() + } + + /// Returns the on-disk size of the level. + pub fn size(&self) -> u64 { + self.0 + .iter() + .flat_map(|x| x.iter()) + .map(Table::file_size) + .sum() + } + + pub fn aggregate_key_range(&self) -> KeyRange { + if self.run_count() == 1 { + #[expect( + clippy::expect_used, + reason = "we check for run_count, so the first run must exist" + )] + self.runs + .first() + .expect("should exist") + .aggregate_key_range() + } else { + let key_ranges = self + .iter() + .map(|x| Run::aggregate_key_range(x)) + .collect::>(); + + KeyRange::aggregate(key_ranges.iter()) + } + } +} + +pub struct VersionInner { + /// The version's ID + id: VersionId, + + tree_type: TreeType, + + /// The individual LSM-tree levels which consist of runs of tables + levels: Vec, + + // NOTE: We purposefully use Arc<_> to avoid deep cloning the blob files again and again + // + // Changing the value log tends to happen way less often than other modifications to the + // LSM-tree + // + /// Blob files for large values (value log) + #[doc(hidden)] + pub blob_files: Arc, + + /// Blob file fragmentation + gc_stats: Arc, +} + +/// A version is an immutable, point-in-time view of a tree's structure +/// +/// Any time a table is created or deleted, a new version is created. +#[derive(Clone)] +pub struct Version { + inner: Arc, +} + +impl std::ops::Deref for Version { + type Target = VersionInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +// TODO: impl using generics so we can easily unit test Version transformation functions +impl Version { + /// Returns the initial tree type. + pub fn tree_type(&self) -> TreeType { + self.tree_type + } + + /// Returns the version ID. + pub fn id(&self) -> VersionId { + self.id + } + + pub fn gc_stats(&self) -> &FragmentationMap { + &self.gc_stats + } + + pub fn l0(&self) -> &Level { + #[expect(clippy::expect_used)] + self.levels.first().expect("L0 should exist") + } + + #[must_use] + pub fn level_is_busy(&self, idx: usize, hidden_set: &HiddenSet) -> bool { + self.level(idx).is_some_and(|level| { + level + .iter() + .flat_map(|run| run.iter()) + .any(|table| hidden_set.is_hidden(table.id())) + }) + } + + /// Creates a new empty version. + pub fn new(id: VersionId, tree_type: TreeType) -> Self { + let levels = (0..DEFAULT_LEVEL_COUNT).map(|_| Level::empty()).collect(); + + Self { + inner: Arc::new(VersionInner { + id, + tree_type, + levels, + blob_files: Arc::default(), + gc_stats: Arc::default(), + }), + } + } + + pub(crate) fn from_recovery( + recovery: Recovery, + tables: &[Table], + blob_files: &[BlobFile], + ) -> crate::Result { + let version_levels = recovery + .table_ids + .iter() + .map(|level| { + let level_runs = level + .iter() + .map(|run| { + let run_tables = run + .iter() + .map(|table| { + tables + .iter() + .find(|x| x.id() == table.id) + .cloned() + .ok_or(crate::Error::Unrecoverable) + }) + .collect::>>()?; + + Ok(Arc::new( + #[expect( + clippy::expect_used, + reason = "empty runs should not exist, so there should not be any empty persisted runs" + )] + Run::new(run_tables).expect("persisted runs should not be empty"), + )) + }) + .collect::>>()?; + + Ok(Level::from_runs(level_runs)) + }) + .collect::>>()?; + + Ok(Self::from_levels( + recovery.curr_version_id, + recovery.tree_type, + version_levels, + BlobFileList::new(blob_files.iter().cloned().map(|bf| (bf.id(), bf)).collect()), + recovery.gc_stats, + )) + } + + /// Creates a new pre-populated version. + pub fn from_levels( + id: VersionId, + tree_type: TreeType, + levels: Vec, + blob_files: BlobFileList, + gc_stats: FragmentationMap, + ) -> Self { + Self { + inner: Arc::new(VersionInner { + id, + tree_type, + levels, + blob_files: Arc::new(blob_files), + gc_stats: Arc::new(gc_stats), + }), + } + } + + /// Returns the number of levels. + pub fn level_count(&self) -> usize { + self.levels.len() + } + + /// Returns an iterator through all levels. + pub fn iter_levels(&self) -> impl Iterator { + self.levels.iter() + } + + /// Returns the number of tables in all levels. + pub fn table_count(&self) -> usize { + self.iter_levels().map(|x| x.table_count()).sum() + } + + pub fn blob_file_count(&self) -> usize { + self.blob_files.len() + } + + /// Returns an iterator over all tables. + pub fn iter_tables(&self) -> impl Iterator { + self.levels + .iter() + .flat_map(|x| x.iter()) + .flat_map(|x| x.iter()) + } + + pub(crate) fn get_table(&self, id: TableId) -> Option<&Table> { + self.iter_tables().find(|x| x.metadata.id == id) + } + + /// Gets the n-th level. + pub fn level(&self, n: usize) -> Option<&Level> { + self.levels.get(n) + } + + /// Creates a new version with the additional run added to the "top" of L0. + pub fn with_new_l0_run( + &self, + run: &[Table], + blob_files: Option<&[BlobFile]>, + diff: Option, + ) -> Self { + let id = self.id + 1; + + let mut levels = vec![]; + + // L0 + levels.push({ + // Copy-on-write the first level with new run at top + + #[expect(clippy::expect_used, reason = "L0 always exists")] + let l0 = self.levels.first().expect("L0 should always exist"); + + let prev_runs = l0 + .runs + .iter() + .map(|run| { + let run: Run<_> = run.deref().clone(); + run + }) + .collect::>(); + + let mut runs = Vec::with_capacity(prev_runs.len() + 1); + + if let Some(run) = Run::new(run.to_vec()) { + runs.push(run); + } + + runs.extend(prev_runs); + + let runs = optimize_runs(runs); + + Level::from_runs(runs.into_iter().map(Arc::new).collect()) + }); + + // L1+ + levels.extend(self.levels.iter().skip(1).cloned()); + + // Value log + let value_log = if let Some(blob_files) = blob_files { + let mut copy = self.blob_files.deref().clone(); + copy.extend(blob_files.iter().cloned().map(|bf| (bf.id(), bf))); + copy.into() + } else { + self.blob_files.clone() + }; + + let gc_stats = if let Some(diff) = diff { + let mut copy = self.gc_stats.deref().clone(); + diff.merge_into(&mut copy); + copy.prune(&value_log); + Arc::new(copy) + } else { + self.gc_stats.clone() + }; + + Self { + inner: Arc::new(VersionInner { + id, + tree_type: self.tree_type, + levels, + blob_files: value_log, + gc_stats, + }), + } + } + + /// Returns a new version with a list of tables removed. + /// + /// The table files are not immediately deleted, this is handled by the version system's free list. + pub fn with_dropped( + &self, + ids: &[TableId], + dropped_blob_files: &mut Vec, + ) -> crate::Result { + let id = self.id + 1; + + let mut levels = vec![]; + + let mut dropped_tables: Vec
= vec![]; + + for level in &self.levels { + let runs = level + .runs + .iter() + .map(|run| { + // TODO: don't clone Arc inner if we don't need to modify + let mut run: Run<_> = run.deref().clone(); + + let removed_tables = run + .inner_mut() + .extract_if(.., |x| ids.contains(&x.metadata.id)); + + dropped_tables.extend(removed_tables); + + run + }) + .filter(|x| !x.is_empty()) + .collect::>(); + + let runs = optimize_runs(runs); + + levels.push(Level::from_runs(runs.into_iter().map(Arc::new).collect())); + } + + let gc_stats = if dropped_tables.is_empty() { + self.gc_stats.clone() + } else { + let mut copy = self.gc_stats.deref().clone(); + + for table in &dropped_tables { + let linked_blob_files = table.list_blob_file_references()?.unwrap_or_default(); + + for blob_file in linked_blob_files { + copy.entry(blob_file.blob_file_id) + .and_modify(|counter| { + counter.bytes += blob_file.bytes; + counter.len += blob_file.len; + }) + .or_insert_with(|| { + FragmentationEntry::new( + blob_file.len, + blob_file.bytes, + blob_file.on_disk_bytes, + ) + }); + } + } + + Arc::new(copy) + }; + + let value_log = if dropped_tables.is_empty() { + self.blob_files.clone() + } else { + let mut copy = self.blob_files.deref().clone(); + dropped_blob_files.extend(copy.prune_dead(&gc_stats)); + Arc::new(copy) + }; + + Ok(Self { + inner: Arc::new(VersionInner { + id, + tree_type: self.tree_type, + levels, + blob_files: value_log, + gc_stats, + }), + }) + } + + pub fn with_merge( + &self, + old_ids: &[TableId], + new_tables: &[Table], + dest_level: usize, + diff: Option, + new_blob_files: Vec, + blob_files_to_drop: &HashSet, + ) -> Self { + let id = self.id + 1; + + let mut levels = vec![]; + + for (level_idx, level) in self.levels.iter().enumerate() { + let mut runs = level + .runs + .iter() + .map(|run| { + // TODO: don't clone Arc inner if we don't need to modify + let mut run: Run<_> = run.deref().clone(); + run.retain(|x| !old_ids.contains(&x.metadata.id)); + run + }) + .filter(|x| !x.is_empty()) + .collect::>(); + + if level_idx == dest_level { + if let Some(run) = Run::new(new_tables.to_vec()) { + runs.insert(0, run); + } + } + + let runs = optimize_runs(runs); + + levels.push(Level::from_runs(runs.into_iter().map(Arc::new).collect())); + } + + let has_diff = diff.is_some(); + + let value_log = if has_diff || !new_blob_files.is_empty() || !blob_files_to_drop.is_empty() + { + let mut copy = self.blob_files.deref().clone(); + + for blob_file in new_blob_files { + copy.insert(blob_file.id(), blob_file); + } + + for &id in blob_files_to_drop { + copy.remove(id); + } + + Arc::new(copy) + } else { + self.blob_files.clone() + }; + + let gc_stats = if has_diff || !blob_files_to_drop.is_empty() { + let mut copy = self.gc_stats.deref().clone(); + + if let Some(diff) = diff { + diff.merge_into(&mut copy); + } + + copy.prune(&value_log); + + Arc::new(copy) + } else { + self.gc_stats.clone() + }; + + Self { + inner: Arc::new(VersionInner { + id, + tree_type: self.tree_type, + levels, + blob_files: value_log, + gc_stats, + }), + } + } + + pub fn with_moved(&self, ids: &[TableId], dest_level: usize) -> Self { + let id = self.id + 1; + + let affected_tables = self + .iter_tables() + .filter(|x| ids.contains(&x.id())) + .cloned() + .collect::>(); + + assert_eq!(affected_tables.len(), ids.len(), "invalid table IDs"); + + let mut levels = vec![]; + + for (level_idx, level) in self.levels.iter().enumerate() { + let mut runs = level + .runs + .iter() + .map(|run| { + // TODO: don't clone Arc inner if we don't need to modify + let mut run: Run<_> = run.deref().clone(); + run.retain(|x| !ids.contains(&x.metadata.id)); + run + }) + .filter(|x| !x.is_empty()) + .collect::>(); + + if level_idx == dest_level { + if let Some(run) = Run::new(affected_tables.clone()) { + runs.insert(0, run); + } + } + + let runs = optimize_runs(runs); + + levels.push(Level::from_runs(runs.into_iter().map(Arc::new).collect())); + } + + Self { + inner: Arc::new(VersionInner { + id, + tree_type: self.tree_type, + levels, + blob_files: self.blob_files.clone(), + gc_stats: self.gc_stats.clone(), + }), + } + } +} + +impl Version { + pub(crate) fn encode_into( + &self, + writer: &mut sfa::Writer, + ) -> Result<(), crate::Error> { + use crate::FormatVersion; + use byteorder::{LittleEndian, WriteBytesExt}; + use std::io::Write; + + // + // Manifest + // + + writer.start("format_version")?; + writer.write_u8(FormatVersion::V4.into())?; + + writer.start("crate_version")?; + writer.write_all(env!("CARGO_PKG_VERSION").as_bytes())?; + + writer.start("tree_type")?; + writer.write_u8(self.tree_type.into())?; + + writer.start("level_count")?; + #[expect( + clippy::cast_possible_truncation, + reason = "level count is bounded by 255" + )] + writer.write_u8(self.level_count() as u8)?; + + writer.start("filter_hash_type")?; + writer.write_u8(u8::from(ChecksumType::Xxh3))?; + + // + // Levels + // + + writer.start("tables")?; + + // Level count + #[expect( + clippy::cast_possible_truncation, + reason = "there are always less than 256 levels" + )] + writer.write_u8(self.level_count() as u8)?; + + for level in self.iter_levels() { + // Run count + #[expect( + clippy::cast_possible_truncation, + reason = "there are always less than 256 runs" + )] + writer.write_u8(level.len() as u8)?; + + for run in level.iter() { + // Table count + #[expect( + clippy::cast_possible_truncation, + reason = "there are always less than 4 billion tables in a run" + )] + writer.write_u32::(run.len() as u32)?; + + // Tables + for table in run.iter() { + writer.write_u64::(table.id())?; + writer.write_u8(0)?; // Checksum type, 0 = XXH3 + writer.write_u128::(table.checksum().into_u128())?; + writer.write_u64::(table.global_seqno())?; + } + } + } + + writer.start("blob_files")?; + + // Blob file count + #[expect( + clippy::cast_possible_truncation, + reason = "there are always less than 4 billion blob files" + )] + writer.write_u32::(self.blob_files.len() as u32)?; + + for file in self.blob_files.iter() { + writer.write_u64::(file.id())?; + writer.write_u8(0)?; // Checksum type, 0 = XXH3 + writer.write_u128::(file.0.checksum.into_u128())?; + } + + writer.start("blob_gc_stats")?; + + self.gc_stats.encode_into(writer)?; + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/version/optimize.rs b/crates/lsm-tree/src/version/optimize.rs new file mode 100644 index 000000000..573955136 --- /dev/null +++ b/crates/lsm-tree/src/version/optimize.rs @@ -0,0 +1,161 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::run::Ranged; +use crate::version::Run; + +pub fn optimize_runs(runs: Vec>) -> Vec> { + if runs.len() <= 1 { + runs + } else { + let mut new_runs: Vec> = Vec::new(); + + for run in runs.iter().rev() { + 'run: for table in run.iter().rev() { + for existing_run in new_runs.iter_mut().rev() { + if existing_run + .iter() + .all(|x| !table.key_range().overlaps_with_key_range(x.key_range())) + { + existing_run.push(table.clone()); + continue 'run; + } + } + + #[expect( + clippy::expect_used, + reason = "we pass in a table, so the run cannot be None" + )] + new_runs.insert( + 0, + Run::new(vec![table.clone()]).expect("run should not be empty"), + ); + } + } + + new_runs + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::KeyRange; + use test_log::test; + + #[derive(Clone, Debug, Eq, PartialEq)] + struct FakeTable { + id: u64, + key_range: KeyRange, + } + + impl Ranged for FakeTable { + fn key_range(&self) -> &KeyRange { + &self.key_range + } + } + + fn s(id: u64, min: &str, max: &str) -> FakeTable { + FakeTable { + id, + key_range: KeyRange::new((min.as_bytes().into(), max.as_bytes().into())), + } + } + + #[test] + fn optimize_runs_empty() { + let runs = vec![]; + let runs = optimize_runs::(runs); + + assert_eq!(Vec::>::new(), &*runs); + } + + #[test] + fn optimize_runs_one() { + let runs = vec![Run::new(vec![s(0, "a", "b")]).unwrap()]; + let runs = optimize_runs::(runs); + + assert_eq!(vec![Run::new(vec![s(0, "a", "b")]).unwrap()], &*runs); + } + + #[test] + fn optimize_runs_two_overlap() { + let runs = vec![ + Run::new(vec![s(0, "a", "b")]).unwrap(), + Run::new(vec![s(1, "a", "b")]).unwrap(), + ]; + let runs = optimize_runs::(runs); + + assert_eq!( + vec![ + Run::new(vec![s(0, "a", "b")]).unwrap(), + Run::new(vec![s(1, "a", "b")]).unwrap(), + ], + &*runs + ); + } + + #[test] + fn optimize_runs_two_overlap_2() { + let runs = vec![ + Run::new(vec![s(0, "a", "z")]).unwrap(), + Run::new(vec![s(1, "c", "f")]).unwrap(), + ]; + let runs = optimize_runs::(runs); + + assert_eq!( + vec![ + Run::new(vec![s(0, "a", "z")]).unwrap(), + Run::new(vec![s(1, "c", "f")]).unwrap(), + ], + &*runs + ); + } + + #[test] + fn optimize_runs_two_overlap_3() { + let runs = vec![ + Run::new(vec![s(0, "c", "f")]).unwrap(), + Run::new(vec![s(1, "a", "z")]).unwrap(), + ]; + let runs = optimize_runs::(runs); + + assert_eq!( + vec![ + Run::new(vec![s(0, "c", "f")]).unwrap(), + Run::new(vec![s(1, "a", "z")]).unwrap() + ], + &*runs + ); + } + + #[test] + fn optimize_runs_two_disjoint() { + let runs = vec![ + Run::new(vec![s(0, "a", "c")]).unwrap(), + Run::new(vec![s(1, "d", "f")]).unwrap(), + ]; + let runs = optimize_runs::(runs); + + assert_eq!( + vec![Run::new(vec![s(0, "a", "c"), s(1, "d", "f")]).unwrap()], + &*runs, + ); + } + + #[test] + fn optimize_runs_two_disjoint_2() { + let runs = vec![ + Run::new(vec![s(1, "d", "f")]).unwrap(), + Run::new(vec![s(0, "a", "c")]).unwrap(), + ]; + let runs = optimize_runs::(runs); + + assert_eq!( + vec![Run::new(vec![s(0, "a", "c"), s(1, "d", "f")]).unwrap()], + &*runs, + ); + } +} diff --git a/crates/lsm-tree/src/version/persist.rs b/crates/lsm-tree/src/version/persist.rs new file mode 100644 index 000000000..299460eeb --- /dev/null +++ b/crates/lsm-tree/src/version/persist.rs @@ -0,0 +1,80 @@ +use crate::{ + checksum::ChecksummedWriter, + file::{fsync_directory, retry_transient_io, rewrite_atomic, CURRENT_VERSION_FILE}, + version::Version, +}; +use byteorder::{LittleEndian, WriteBytesExt}; +use std::{ + io::{BufWriter, Write}, + path::Path, +}; + +pub fn persist_version(folder: &Path, version: &Version) -> crate::Result<()> { + log::trace!( + "Persisting version {} in {}", + version.id(), + folder.display(), + ); + + let path = folder.join(format!("v{}", version.id())); + let file = retry_transient_io(|| std::fs::File::create(&path))?; + let writer = BufWriter::new(&file); + let mut writer = ChecksummedWriter::new(writer); + + { + let mut writer = sfa::Writer::from_writer(&mut writer); + + version.encode_into(&mut writer)?; + + writer.finish().map_err(|e| match e { + sfa::Error::Io(e) => crate::Error::from(e), + _ => unreachable!(), + })?; + } + + writer.flush()?; + + let checksum = writer.checksum(); + + drop(writer); + + file.sync_all()?; + + // IMPORTANT: fsync folder on Unix + fsync_directory(folder)?; + + let mut current_file_content = vec![]; + current_file_content.write_u64::(version.id())?; + current_file_content.write_u128::(checksum.into_u128())?; + current_file_content.write_u8(0)?; // 0 = xxh3 + + rewrite_atomic(&folder.join(CURRENT_VERSION_FILE), ¤t_file_content)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TreeType; + use test_log::test; + + #[test] + fn version_persist_replaces_orphaned_file() -> crate::Result<()> { + let dir = tempfile::tempdir()?; + let version = Version::new(0, TreeType::Standard); + + // Simulates the leftover of a persist that failed midway + std::fs::write(dir.path().join("v0"), b"partial")?; + + persist_version(dir.path(), &version)?; + + assert_ne!( + b"partial".as_slice(), + &*std::fs::read(dir.path().join("v0"))? + ); + assert!(dir.path().join(CURRENT_VERSION_FILE).try_exists()?); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/version/recovery.rs b/crates/lsm-tree/src/version/recovery.rs new file mode 100644 index 000000000..515d5c9be --- /dev/null +++ b/crates/lsm-tree/src/version/recovery.rs @@ -0,0 +1,160 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + coding::Decode, file::CURRENT_VERSION_FILE, version::VersionId, vlog::BlobFileId, Checksum, + SeqNo, TableId, TreeType, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::path::Path; + +pub fn get_current_version(folder: &std::path::Path) -> crate::Result { + use byteorder::{LittleEndian, ReadBytesExt}; + + std::fs::File::open(folder.join(CURRENT_VERSION_FILE)) + .and_then(|mut f| f.read_u64::()) + .map_err(Into::into) +} + +pub struct RecoveredTable { + pub id: TableId, + pub checksum: Checksum, + pub global_seqno: SeqNo, +} + +pub struct Recovery { + pub tree_type: TreeType, + pub curr_version_id: VersionId, + pub table_ids: Vec>>, + pub blob_file_ids: Vec<(BlobFileId, Checksum)>, + pub gc_stats: crate::blob_tree::FragmentationMap, +} + +pub fn recover(folder: &Path) -> crate::Result { + let curr_version_id = get_current_version(folder)?; + let version_file_path = folder.join(format!("v{curr_version_id}")); + + // TODO: maybe validate current version using the checksum in "current" + + log::info!( + "Recovering current manifest at {}", + version_file_path.display(), + ); + + let reader = sfa::Reader::new(&version_file_path)?; + let toc = reader.toc(); + + // // TODO: vvv move into Version::decode vvv + let mut levels = vec![]; + + { + let mut reader = toc + .section(b"tables") + .ok_or(crate::Error::Unrecoverable) + .inspect_err(|_| { + log::error!("tables section not found in version #{curr_version_id} - maybe the file is corrupted?"); + })? + .buf_reader(&version_file_path)?; + + let level_count = reader.read_u8()?; + + for _ in 0..level_count { + let mut level = vec![]; + let run_count = reader.read_u8()?; + + for _ in 0..run_count { + let mut run = vec![]; + let table_count = reader.read_u32::()?; + + for _ in 0..table_count { + let id = reader.read_u64::()?; + let checksum_type = reader.read_u8()?; + + if checksum_type != 0 { + return Err(crate::Error::InvalidTag(("ChecksumType", checksum_type))); + } + + let checksum = reader.read_u128::()?; + let checksum = Checksum::from_raw(checksum); + + let global_seqno = reader.read_u64::()?; + + run.push(RecoveredTable { + id, + checksum, + global_seqno, + }); + } + + level.push(run); + } + + levels.push(level); + } + } + + let blob_file_ids = { + let mut reader = toc + .section(b"blob_files") + .ok_or(crate::Error::Unrecoverable) + .inspect_err(|_| { + log::error!("blob_files section not found in version #{curr_version_id} - maybe the file is corrupted?"); + })? + .buf_reader(&version_file_path)?; + + let blob_file_count = reader.read_u32::()?; + let mut blob_file_ids = Vec::with_capacity(blob_file_count as usize); + + for _ in 0..blob_file_count { + let id = reader.read_u64::()?; + + let checksum_type = reader.read_u8()?; + + if checksum_type != 0 { + return Err(crate::Error::InvalidTag(("ChecksumType", checksum_type))); + } + + let checksum = reader.read_u128::()?; + let checksum = Checksum::from_raw(checksum); + + blob_file_ids.push((id, checksum)); + } + + blob_file_ids.sort_by_key(|(id, _)| *id); + blob_file_ids + }; + + debug_assert!(blob_file_ids.is_sorted_by_key(|(id, _)| id)); + + let gc_stats = { + let mut reader = toc + .section(b"blob_gc_stats") + .ok_or(crate::Error::Unrecoverable) + .inspect_err(|_| { + log::error!("blob_gc_stats section not found in version #{curr_version_id} - maybe the file is corrupted?"); + })? + .buf_reader(&version_file_path)?; + + crate::blob_tree::FragmentationMap::decode_from(&mut reader)? + }; + + Ok(Recovery { + tree_type: { + let byte = toc.section(b"tree_type").ok_or(crate::Error::Unrecoverable) + .inspect_err(|_|{ + log::error!("tree_type section not found in version #{curr_version_id} - maybe the file is corrupted?"); + })? + .buf_reader( + &version_file_path + )? + .read_u8()?; + + TreeType::try_from(byte).map_err(|()| crate::Error::InvalidHeader("TreeType"))? + }, + curr_version_id, + table_ids: levels, + blob_file_ids, + gc_stats, + }) +} diff --git a/crates/lsm-tree/src/version/run.rs b/crates/lsm-tree/src/version/run.rs new file mode 100644 index 000000000..686416ec9 --- /dev/null +++ b/crates/lsm-tree/src/version/run.rs @@ -0,0 +1,422 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::KeyRange; +use std::ops::{Bound, RangeBounds}; + +pub trait Ranged { + fn key_range(&self) -> &KeyRange; +} + +/// A disjoint run of tables +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Run(Vec); + +impl std::ops::Deref for Run { + type Target = [T]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Run { + pub fn new(items: Vec) -> Option { + if items.is_empty() { + None + } else { + Some(Self(items)) + } + } + + pub fn inner_mut(&mut self) -> &mut Vec { + &mut self.0 + } + + pub fn push(&mut self, item: T) { + self.0.push(item); + + self.0 + .sort_by(|a, b| a.key_range().min().cmp(b.key_range().min())); + } + + pub fn extend(&mut self, items: Vec) { + self.0.extend(items); + } + + pub fn retain(&mut self, f: F) + where + F: FnMut(&T) -> bool, + { + self.0.retain(f); + } + + pub fn remove(&mut self, idx: usize) -> T { + self.0.remove(idx) + } + + /// Returns the table that may possibly contains the given key. + pub fn get_for_key(&self, key: &[u8]) -> Option<&T> { + let idx = self.partition_point(|x| x.key_range().max() < &key); + + self.0.get(idx).filter(|x| x.key_range().min() <= &key) + } + + /// Returns the run's key range. + pub fn aggregate_key_range(&self) -> KeyRange { + #[expect(clippy::expect_used, reason = "by definition, runs are never empty")] + let lo = self.first().expect("run should never be empty"); + + #[expect(clippy::expect_used, reason = "by definition, runs are never empty")] + let hi = self.last().expect("run should never be empty"); + + KeyRange::new((lo.key_range().min().clone(), hi.key_range().max().clone())) + } + + /// Returns the sub slice of tables in the run that have + /// a key range overlapping the input key range. + pub fn get_overlapping<'a>(&'a self, key_range: &'a KeyRange) -> &'a [T] { + let range = key_range.min()..=key_range.max(); + + let Some((lo, hi)) = self.range_overlap_indexes::(&range) else { + return &[]; + }; + + self.get(lo..=hi).unwrap_or_default() + } + + /// Returns the sub slice of tables of tables in the run that have + /// a key range fully contained in the input key range. + pub fn get_contained<'a>(&'a self, key_range: &KeyRange) -> &'a [T] { + fn trim_slice(s: &[T], pred: F) -> &[T] + where + F: Fn(&T) -> bool, + { + // find first index where pred holds + let start = s.iter().position(&pred).unwrap_or(s.len()); + + // find last index where pred holds + let end = s.iter().rposition(&pred).map_or(start, |i| i + 1); + + s.get(start..end).expect("should be in range") + } + + let range = key_range.min()..=key_range.max(); + + let Some((lo, hi)) = self.range_overlap_indexes::(&range) else { + return &[]; + }; + + self.get(lo..=hi) + .map(|slice| trim_slice(slice, |x| key_range.contains_range(x.key_range()))) + .unwrap_or_default() + } + + /// Returns the indexes of the interval [min, max] of tables that overlap with a given range. + pub fn range_overlap_indexes, R: RangeBounds>( + &self, + key_range: &R, + ) -> Option<(usize, usize)> { + let level = &self.0; + + let lo = match key_range.start_bound() { + Bound::Unbounded => 0, + Bound::Included(start_key) => { + level.partition_point(|x| x.key_range().max() < start_key) + } + Bound::Excluded(start_key) => { + level.partition_point(|x| x.key_range().max() <= start_key) + } + }; + + if lo >= level.len() { + return None; + } + + // NOTE: We check for level length above + #[expect(clippy::indexing_slicing)] + let truncated_level = &level[lo..]; + + let hi = match key_range.end_bound() { + Bound::Unbounded => level.len() - 1, + Bound::Included(end_key) => { + // IMPORTANT: We need to add back `lo` because we sliced it off + let idx = lo + truncated_level.partition_point(|x| x.key_range().min() <= end_key); + + if idx == 0 { + return None; + } + + idx.saturating_sub(1) // To avoid underflow + } + Bound::Excluded(end_key) => { + // IMPORTANT: We need to add back `lo` because we sliced it off + let idx = lo + truncated_level.partition_point(|x| x.key_range().min() < end_key); + + if idx == 0 { + return None; + } + + idx.saturating_sub(1) // To avoid underflow + } + }; + + if lo > hi { + return None; + } + + Some((lo, hi)) + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use test_log::test; + + #[derive(Clone)] + struct FakeTable { + id: u64, + key_range: KeyRange, + } + + impl Ranged for FakeTable { + fn key_range(&self) -> &KeyRange { + &self.key_range + } + } + + fn s(id: u64, min: &str, max: &str) -> FakeTable { + FakeTable { + id, + key_range: KeyRange::new((min.as_bytes().into(), max.as_bytes().into())), + } + } + + #[test] + fn run_aggregate_key_range() { + let items = vec![ + s(0, "a", "d"), + s(1, "e", "j"), + s(2, "k", "o"), + s(3, "p", "z"), + ]; + let run = Run(items); + + assert_eq!( + KeyRange::new((b"a".into(), b"z".into())), + run.aggregate_key_range(), + ); + } + + #[test] + fn run_point_lookup() { + let items = vec![ + s(0, "a", "d"), + s(1, "e", "j"), + s(2, "k", "o"), + s(3, "p", "z"), + ]; + let run = Run(items); + + assert_eq!(0, run.get_for_key(b"a").unwrap().id); + assert_eq!(0, run.get_for_key(b"aaa").unwrap().id); + assert_eq!(0, run.get_for_key(b"b").unwrap().id); + assert_eq!(0, run.get_for_key(b"c").unwrap().id); + assert_eq!(0, run.get_for_key(b"d").unwrap().id); + assert_eq!(1, run.get_for_key(b"e").unwrap().id); + assert_eq!(1, run.get_for_key(b"j").unwrap().id); + assert_eq!(2, run.get_for_key(b"k").unwrap().id); + assert_eq!(2, run.get_for_key(b"o").unwrap().id); + assert_eq!(3, run.get_for_key(b"p").unwrap().id); + assert_eq!(3, run.get_for_key(b"z").unwrap().id); + assert!(run.get_for_key(b"zzz").is_none()); + } + + #[test] + fn run_range_culling() { + let items = vec![ + s(0, "a", "d"), + s(1, "e", "j"), + s(2, "k", "o"), + s(3, "p", "z"), + ]; + let run = Run(items); + + assert_eq!(Some((0, 3)), run.range_overlap_indexes::<&[u8], _>(&..)); + assert_eq!( + Some((0, 0)), + run.range_overlap_indexes(&(b"a" as &[u8]..=b"a")) + ); + assert_eq!( + Some((0, 0)), + run.range_overlap_indexes(&(b"a" as &[u8]..=b"b")) + ); + assert_eq!( + Some((0, 0)), + run.range_overlap_indexes(&(b"a" as &[u8]..=b"d")) + ); + assert_eq!( + Some((0, 0)), + run.range_overlap_indexes(&(b"d" as &[u8]..=b"d")) + ); + assert_eq!( + Some((0, 0)), + run.range_overlap_indexes(&(b"a" as &[u8]..b"d")) + ); + assert_eq!( + Some((0, 1)), + run.range_overlap_indexes(&(b"a" as &[u8]..=b"g")) + ); + assert_eq!( + Some((1, 1)), + run.range_overlap_indexes(&(b"j" as &[u8]..=b"j")) + ); + assert_eq!( + Some((0, 3)), + run.range_overlap_indexes(&(b"a" as &[u8]..=b"z")) + ); + assert_eq!( + Some((3, 3)), + run.range_overlap_indexes(&(b"z" as &[u8]..=b"zzz")) + ); + assert_eq!(Some((3, 3)), run.range_overlap_indexes(&(b"z" as &[u8]..))); + assert!(run + .range_overlap_indexes(&(b"zzz" as &[u8]..=b"zzzzzzz")) + .is_none()); + } + + #[test] + fn run_range_contained() { + use crate::TableId; + + let items = vec![ + s(0, "a", "d"), + s(1, "e", "j"), + s(2, "k", "o"), + s(3, "p", "z"), + ]; + let run = Run(items); + + assert_eq!( + &[] as &[TableId], + &*run + .get_contained(&KeyRange::new((b"a".into(), b"a".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0], + &*run + .get_contained(&KeyRange::new((b"a".into(), b"d".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0, 1], + &*run + .get_contained(&KeyRange::new((b"a".into(), b"j".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0, 1], + &*run + .get_contained(&KeyRange::new((b"a".into(), b"k".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0, 1], + &*run + .get_contained(&KeyRange::new((b"a".into(), b"l".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0, 1, 2, 3], + &*run + .get_contained(&KeyRange::new((b"a".into(), b"z".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + } + + #[test] + fn run_range_overlaps() { + let items = vec![ + s(0, "a", "d"), + s(1, "e", "j"), + s(2, "k", "o"), + s(3, "p", "z"), + ]; + let run = Run(items); + + assert_eq!( + &[0], + &*run + .get_overlapping(&KeyRange::new((b"a".into(), b"a".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0], + &*run + .get_overlapping(&KeyRange::new((b"d".into(), b"d".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0], + &*run + .get_overlapping(&KeyRange::new((b"a".into(), b"d".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0, 1], + &*run + .get_overlapping(&KeyRange::new((b"a".into(), b"f".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[0, 1, 2, 3], + &*run + .get_overlapping(&KeyRange::new((b"a".into(), b"zzz".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + + assert_eq!( + &[] as &[u64], + &*run + .get_overlapping(&KeyRange::new((b"zzz".into(), b"zzzz".into()))) + .iter() + .map(|x| x.id) + .collect::>(), + ); + } +} diff --git a/crates/lsm-tree/src/version/super_version.rs b/crates/lsm-tree/src/version/super_version.rs new file mode 100644 index 000000000..2398a45ab --- /dev/null +++ b/crates/lsm-tree/src/version/super_version.rs @@ -0,0 +1,370 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + memtable::Memtable, + tree::sealed::SealedMemtables, + version::{persist_version, Version}, + SeqNo, SequenceNumberCounter, +}; +use std::{collections::VecDeque, path::Path, sync::Arc}; + +/// A super version is a point-in-time snapshot of memtables and a [`Version`] (list of disk files) +#[derive(Clone)] +pub struct SuperVersion { + /// Active memtable that is being written to + #[doc(hidden)] + pub active_memtable: Arc, + + /// Frozen memtables that are being flushed + pub(crate) sealed_memtables: Arc, + + /// Current tree version + pub(crate) version: Version, + + pub(crate) seqno: SeqNo, +} + +pub struct SuperVersions(VecDeque>); + +impl SuperVersions { + pub fn new(version: Version) -> Self { + Self( + vec![Arc::new(SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version, + seqno: 0, + })] + .into(), + ) + } + + pub fn memtable_size_sum(&self) -> u64 { + let mut set = crate::HashMap::default(); + + for super_version in &self.0 { + set.entry(super_version.active_memtable.id) + .and_modify(|bytes| *bytes += super_version.active_memtable.size()) + .or_insert_with(|| super_version.active_memtable.size()); + + for sealed in super_version.sealed_memtables.iter() { + set.entry(sealed.id) + .and_modify(|bytes| *bytes += sealed.size()) + .or_insert_with(|| sealed.size()); + } + } + + set.into_values().sum() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn free_list_len(&self) -> usize { + self.len().saturating_sub(1) + } + + pub fn maintenance(&mut self, folder: &Path, gc_watermark: SeqNo) -> crate::Result<()> { + if gc_watermark == 0 { + return Ok(()); + } + + if self.free_list_len() < 1 { + return Ok(()); + } + + log::trace!("Running manifest GC with watermark={gc_watermark}"); + + if let Some(hi_idx) = self.0.iter().rposition(|x| x.seqno < gc_watermark) { + for _ in 0..hi_idx { + let Some(head) = self.0.front() else { + break; + }; + + log::trace!( + "Removing version #{} (seqno={})", + head.version.id(), + head.seqno, + ); + + let path = folder.join(format!("v{}", head.version.id())); + if path.try_exists()? { + crate::file::retry_transient_io(|| std::fs::remove_file(&path))?; + } + + self.0.pop_front(); + } + } + + log::trace!("Manifest GC done, version length now {}", self.0.len()); + + Ok(()) + } + + /// Modifies the level manifest atomically. + /// + /// The function accepts a transition function that receives the current version + /// and returns a new version. + /// + /// The function takes care of persisting the version changes on disk. + pub(crate) fn upgrade_version crate::Result>( + &mut self, + tree_path: &Path, + f: F, + seqno: &SequenceNumberCounter, + visible_seqno: &SequenceNumberCounter, + ) -> crate::Result<()> { + self.upgrade_version_with_seqno(tree_path, f, seqno.next(), visible_seqno) + } + + /// Like `upgrade_version`, but takes an already-allocated sequence number. + /// + /// This is useful when the seqno must be coordinated with other operations + /// (e.g., bulk ingestion where tables are recovered with the same seqno). + pub(crate) fn upgrade_version_with_seqno< + F: FnOnce(&SuperVersion) -> crate::Result, + >( + &mut self, + tree_path: &Path, + f: F, + seqno: SeqNo, + visible_seqno: &SequenceNumberCounter, + ) -> crate::Result<()> { + let mut next_version = f(&self.latest_version())?; + next_version.seqno = seqno; + log::trace!("Next version seqno={}", next_version.seqno); + + persist_version(tree_path, &next_version.version)?; + self.append_version(next_version); + + visible_seqno.fetch_max(seqno + 1); + + Ok(()) + } + + pub fn append_version(&mut self, version: SuperVersion) { + self.0.push_back(Arc::new(version)); + } + + pub fn replace_latest_version(&mut self, version: SuperVersion) { + if self.0.pop_back().is_some() { + self.0.push_back(Arc::new(version)); + } + } + + pub fn latest_version(&self) -> SuperVersion { + #[expect(clippy::expect_used, reason = "SuperVersion is expected to exist")] + self.0 + .iter() + .last() + .map(|version| version.as_ref().clone()) + .expect("should always have a SuperVersion") + } + + pub(crate) fn get_version_arc_for_snapshot(&self, seqno: SeqNo) -> Arc { + if seqno == 0 { + #[expect(clippy::expect_used, reason = "SuperVersion is expected to exist")] + return self + .0 + .front() + .cloned() + .expect("should always find a SuperVersion"); + } + + let version = self + .0 + .iter() + .rev() + .find(|version| version.seqno < seqno) + .cloned(); + + if version.is_none() { + log::error!("Failed to find a SuperVersion for snapshot with seqno={seqno}"); + log::error!("SuperVersions:"); + + for version in self.0.iter().rev() { + log::error!("-> {}, seqno={}", version.version.id(), version.seqno); + } + } + + #[expect(clippy::expect_used, reason = "SuperVersion is expected to exist")] + version.expect("should always find a SuperVersion") + } + + pub fn get_version_for_snapshot(&self, seqno: SeqNo) -> SuperVersion { + self.get_version_arc_for_snapshot(seqno).as_ref().clone() + } + + #[cfg(test)] + fn from_versions(versions: VecDeque) -> Self { + Self(versions.into_iter().map(Arc::new).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn super_version_gc_above_watermark() -> crate::Result<()> { + let mut history = SuperVersions::from_versions( + vec![ + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 0, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 1, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 2, + }, + ] + .into(), + ); + + history.maintenance(Path::new("."), 0)?; + + assert_eq!(history.free_list_len(), 2); + + Ok(()) + } + + #[test] + fn super_version_gc_below_watermark_simple() -> crate::Result<()> { + let mut history = SuperVersions::from_versions( + vec![ + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 0, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 1, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 2, + }, + ] + .into(), + ); + + history.maintenance(Path::new("."), 3)?; + + assert_eq!(history.len(), 1); + + Ok(()) + } + + #[test] + fn super_version_gc_below_watermark_simple_2() -> crate::Result<()> { + let mut history = SuperVersions::from_versions( + vec![ + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 0, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 1, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 2, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 8, + }, + ] + .into(), + ); + + history.maintenance(Path::new("."), 3)?; + + assert_eq!(history.len(), 2); + + Ok(()) + } + + #[test] + fn super_version_gc_below_watermark_keep() -> crate::Result<()> { + let mut history = SuperVersions::from_versions( + vec![ + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 0, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 8, + }, + ] + .into(), + ); + + history.maintenance(Path::new("."), 3)?; + + assert_eq!(history.len(), 2); + + Ok(()) + } + + #[test] + fn super_version_gc_below_watermark_shadowed() -> crate::Result<()> { + let mut history = SuperVersions::from_versions( + vec![ + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 0, + }, + SuperVersion { + active_memtable: Arc::new(Memtable::new(0)), + sealed_memtables: Arc::default(), + version: Version::new(0, crate::TreeType::Standard), + seqno: 2, + }, + ] + .into(), + ); + + history.maintenance(Path::new("."), 3)?; + + assert_eq!(history.len(), 1); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/vlog/accessor.rs b/crates/lsm-tree/src/vlog/accessor.rs new file mode 100644 index 000000000..f246ff7d7 --- /dev/null +++ b/crates/lsm-tree/src/vlog/accessor.rs @@ -0,0 +1,56 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + version::BlobFileList, + vlog::{blob_file::reader::Reader, ValueHandle}, + Cache, GlobalTableId, TreeId, UserValue, +}; +use std::{fs::File, path::Path, sync::Arc}; + +pub struct Accessor<'a>(&'a BlobFileList); + +impl<'a> Accessor<'a> { + pub fn new(blob_files: &'a BlobFileList) -> Self { + Self(blob_files) + } + + pub fn get( + &self, + tree_id: TreeId, + base_path: &Path, + key: &[u8], + vhandle: &ValueHandle, + cache: &Cache, + ) -> crate::Result> { + if let Some(value) = cache.get_blob(tree_id, vhandle) { + return Ok(Some(value)); + } + + let Some(blob_file) = self.0.get(vhandle.blob_file_id) else { + return Ok(None); + }; + + let bf_id = GlobalTableId::from((tree_id, blob_file.id())); + + let (file, fd_cache_miss) = + if let Some(cached_fd) = blob_file.file_accessor().access_for_blob_file(&bf_id) { + (cached_fd, false) + } else { + let file = Arc::new(File::open( + base_path.join(vhandle.blob_file_id.to_string()), + )?); + (file, true) + }; + + let value = Reader::new(blob_file, &file).get(key, vhandle)?; + cache.insert_blob(tree_id, vhandle, value.clone()); + + if fd_cache_miss { + blob_file.file_accessor().insert_for_blob_file(bf_id, file); + } + + Ok(Some(value)) + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/merge.rs b/crates/lsm-tree/src/vlog/blob_file/merge.rs new file mode 100644 index 000000000..9a87a420c --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/merge.rs @@ -0,0 +1,203 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::scanner::Scanner as BlobFileScanner; +use crate::vlog::{blob_file::scanner::ScanEntry, BlobFileId}; +use interval_heap::IntervalHeap; +use std::cmp::Reverse; + +type IteratorIndex = usize; + +#[derive(Debug)] +struct IteratorValue { + index: IteratorIndex, + scan_entry: ScanEntry, + blob_file_id: BlobFileId, +} + +impl PartialEq for IteratorValue { + fn eq(&self, other: &Self) -> bool { + self.scan_entry.key == other.scan_entry.key + } +} +impl Eq for IteratorValue {} + +impl PartialOrd for IteratorValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for IteratorValue { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (&self.scan_entry.key, Reverse(&self.scan_entry.seqno)) + .cmp(&(&other.scan_entry.key, Reverse(&other.scan_entry.seqno))) + } +} + +/// Interleaves multiple blob file readers into a single, sorted stream +pub struct MergeScanner { + readers: Vec, + heap: IntervalHeap, +} + +impl MergeScanner { + /// Initializes a new merging reader + pub fn new(readers: Vec) -> Self { + let heap = IntervalHeap::with_capacity(readers.len()); + Self { readers, heap } + } + + fn advance_reader(&mut self, idx: usize) -> crate::Result<()> { + #[expect(clippy::indexing_slicing, reason = "we trust the caller")] + let reader = &mut self.readers[idx]; + + if let Some(value) = reader.next() { + let scan_entry = value?; + let blob_file_id = reader.blob_file_id; + + self.heap.push(IteratorValue { + index: idx, + blob_file_id, + scan_entry, + }); + } + + Ok(()) + } + + fn push_next(&mut self) -> crate::Result<()> { + for idx in 0..self.readers.len() { + self.advance_reader(idx)?; + } + + Ok(()) + } +} + +impl Iterator for MergeScanner { + type Item = crate::Result<(ScanEntry, BlobFileId)>; + + fn next(&mut self) -> Option { + if self.heap.is_empty() { + fail_iter!(self.push_next()); + } + + if let Some(head) = self.heap.pop_min() { + fail_iter!(self.advance_reader(head.index)); + return Some(Ok((head.scan_entry, head.blob_file_id))); + } + + None + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::super::scanner::Scanner; + use super::*; + use crate::{vlog::blob_file::writer::Writer as BlobFileWriter, Slice}; + use tempfile::tempdir; + use test_log::test; + + #[test] + fn blob_file_merger_seqno() -> crate::Result<()> { + let dir = tempdir()?; + + let blob_file_path = dir.path().join("0"); + { + { + let mut writer = BlobFileWriter::new(&blob_file_path, 0, 0)?; + + writer.write(b"a", 1, &b"1".repeat(100))?; + writer.write(b"a", 0, &b"0".repeat(100))?; + + writer.finish()?; + } + } + + { + let mut merger = MergeScanner::new(vec![Scanner::new(&blob_file_path, 0)?]); + + assert_eq!( + (Slice::from(b"a"), Slice::from(b"1".repeat(100))), + merger + .next() + .map(|result| result.map(|(entry, _)| (entry.key, entry.value))) + .unwrap()?, + ); + assert_eq!( + (Slice::from(b"a"), Slice::from(b"0".repeat(100))), + merger + .next() + .map(|result| result.map(|(entry, _)| (entry.key, entry.value))) + .unwrap()?, + ); + + assert!(merger.next().is_none()); + } + + Ok(()) + } + + #[test] + fn blob_file_merger() -> crate::Result<()> { + let dir = tempdir()?; + + let blob_file_0_path = dir.path().join("0"); + let blob_file_1_path = dir.path().join("1"); + + { + let keys = [b"a", b"c", b"e"]; + + { + let mut writer = BlobFileWriter::new(&blob_file_0_path, 0, 0)?; + + for key in keys { + writer.write(key, 0, &key.repeat(100))?; + } + + writer.finish()?; + } + } + + { + let keys = [b"b", b"d"]; + + { + let mut writer = BlobFileWriter::new(&blob_file_1_path, 1, 0)?; + + for key in keys { + writer.write(key, 1, &key.repeat(100))?; + } + + writer.finish()?; + } + } + + { + let mut merger = MergeScanner::new(vec![ + Scanner::new(&blob_file_0_path, 0)?, + Scanner::new(&blob_file_1_path, 1)?, + ]); + + let merged_keys = [b"a", b"b", b"c", b"d", b"e"]; + + for key in merged_keys { + assert_eq!( + (Slice::from(key), Slice::from(key.repeat(100))), + merger + .next() + .map(|result| result.map(|(entry, _)| (entry.key, entry.value))) + .unwrap()?, + ); + } + + assert!(merger.next().is_none()); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/meta.rs b/crates/lsm-tree/src/vlog/blob_file/meta.rs new file mode 100644 index 000000000..821e77c37 --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/meta.rs @@ -0,0 +1,186 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + checksum::ChecksumType, + coding::{Decode, Encode}, + table::{Block, DataBlock}, + vlog::BlobFileId, + CompressionType, InternalValue, KeyRange, SeqNo, Slice, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::io::{Read, Write}; + +macro_rules! read_u64 { + ($block:expr, $name:expr) => {{ + let bytes = $block + .point_read($name, SeqNo::MAX) + .unwrap_or_else(|| panic!("meta property {:?} should exist", $name)); + + let mut bytes = &bytes.value[..]; + bytes.read_u64::()? + }}; +} + +macro_rules! read_u128 { + ($block:expr, $name:expr) => {{ + let bytes = $block + .point_read($name, SeqNo::MAX) + .unwrap_or_else(|| panic!("meta property {:?} should exist", $name)); + + let mut bytes = &bytes.value[..]; + bytes.read_u128::()? + }}; +} + +pub const METADATA_HEADER_MAGIC: &[u8] = b"META"; + +#[derive(Debug, PartialEq, Eq)] +pub struct Metadata { + pub id: BlobFileId, + + pub created_at: u128, + + /// Number of KV-pairs in the blob file + pub item_count: u64, + + /// compressed size in bytes (on disk) (without metadata or trailer) + pub total_compressed_bytes: u64, + + /// true size in bytes (if no compression were used) + pub total_uncompressed_bytes: u64, + + /// Key range + pub key_range: KeyRange, + + /// Compression type used for all blobs in this file + pub compression: CompressionType, +} + +impl Metadata { + pub fn encode_into(&self, writer: &mut W) -> crate::Result<()> { + fn meta(key: &str, value: &[u8]) -> InternalValue { + InternalValue::from_components(key, value, 0, crate::ValueType::Value) + } + + // Write header + writer.write_all(METADATA_HEADER_MAGIC)?; + + #[rustfmt::skip] + let meta_items = [ + meta("blob_file_version", &[0x3]), + meta("checksum_type", &[u8::from(ChecksumType::Xxh3)]), + meta("compression", &self.compression.encode_into_vec()), + meta("crate_version", env!("CARGO_PKG_VERSION").as_bytes()), + meta("created_at", &self.created_at.to_le_bytes()), + meta("file_size", &self.total_compressed_bytes.to_le_bytes()), + meta("id", &self.id.to_le_bytes()), + meta("item_count", &self.item_count.to_le_bytes()), + meta("key#max", self.key_range.max()), + meta("key#min", self.key_range.min()), + meta("uncompressed_size", &self.total_uncompressed_bytes.to_le_bytes()), + ]; + + // NOTE: Just to make sure the items are definitely sorted + #[cfg(debug_assertions)] + { + let is_sorted = meta_items.iter().is_sorted_by_key(|kv| &kv.key); + assert!(is_sorted, "meta items not sorted correctly"); + } + + // TODO: no binary index + let buf = DataBlock::encode_into_vec(&meta_items, 1, 0.0)?; + + Block::write_into( + writer, + &buf, + crate::table::block::BlockType::Meta, + CompressionType::None, + )?; + + Ok(()) + } + + pub fn from_slice(slice: &Slice) -> crate::Result { + let reader = &mut &slice[..]; + + // Check header + let mut magic = [0u8; METADATA_HEADER_MAGIC.len()]; + reader.read_exact(&mut magic)?; + + if magic != METADATA_HEADER_MAGIC { + return Err(crate::Error::InvalidHeader("BlobFileMeta")); + } + + // TODO: Block::from_slice + let block = Block::from_reader(reader, CompressionType::None)?; + let block = DataBlock::new(block); + + let id = read_u64!(block, b"id"); + let created_at = read_u128!(block, b"created_at"); + let item_count = read_u64!(block, b"item_count"); + let file_size = read_u64!(block, b"file_size"); + let total_uncompressed_bytes = read_u64!(block, b"uncompressed_size"); + + let compression = { + #[expect(clippy::expect_used, reason = "compression is expected to exist")] + let bytes = block + .point_read(b"compression", SeqNo::MAX) + .expect("compression should exist"); + + let mut bytes = &bytes.value[..]; + CompressionType::decode_from(&mut bytes)? + }; + + let key_range = KeyRange::new(( + #[expect(clippy::expect_used, reason = "key min is expected to exist")] + block + .point_read(b"key#min", SeqNo::MAX) + .expect("key min should exist") + .value, + #[expect(clippy::expect_used, reason = "key max is expected to exist")] + block + .point_read(b"key#max", SeqNo::MAX) + .expect("key max should exist") + .value, + )); + + Ok(Self { + id, + created_at, + compression, + item_count, + total_compressed_bytes: file_size, + total_uncompressed_bytes, + key_range, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + #[expect(clippy::unwrap_used)] + fn test_blob_file_meta_roundtrip() { + let meta = Metadata { + id: 0, + created_at: 1_234_567_890, + compression: CompressionType::None, + item_count: 100, + total_compressed_bytes: 1024, + total_uncompressed_bytes: 2048, + key_range: KeyRange::new((b"a".into(), b"z".into())), + }; + + let mut buf = Vec::new(); + meta.encode_into(&mut buf).unwrap(); + let buf = Slice::from(buf); + + let meta2 = Metadata::from_slice(&buf).unwrap(); + assert_eq!(meta, meta2); + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/mod.rs b/crates/lsm-tree/src/vlog/blob_file/mod.rs new file mode 100644 index 000000000..138e5e142 --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/mod.rs @@ -0,0 +1,156 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +pub mod merge; +pub mod meta; +pub mod multi_writer; +pub mod reader; +pub mod scanner; +pub mod writer; + +use crate::{ + blob_tree::FragmentationMap, file_accessor::FileAccessor, vlog::BlobFileId, Checksum, + GlobalTableId, TreeId, +}; +pub use meta::Metadata; +use std::{ + path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc}, +}; + +/// A blob file is an immutable, sorted, contiguous file that contains large key-value pairs (blobs) +#[derive(Debug)] +pub struct Inner { + /// Blob file ID + pub id: BlobFileId, + + pub tree_id: TreeId, + + /// File path + pub path: PathBuf, + + /// Statistics + pub meta: Metadata, + + /// Whether this blob file is deleted (logically) + pub is_deleted: AtomicBool, + + pub checksum: Checksum, + + pub(crate) file_accessor: FileAccessor, +} + +impl Inner { + fn global_id(&self) -> GlobalTableId { + GlobalTableId::from((self.tree_id, self.id)) + } +} + +impl Drop for Inner { + fn drop(&mut self) { + if self.is_deleted.load(std::sync::atomic::Ordering::Acquire) { + log::trace!( + "Cleanup deleted blob file {:?} at {}", + self.id, + self.path.display(), + ); + + if let Err(e) = std::fs::remove_file(&*self.path) { + log::warn!( + "Failed to cleanup deleted blob file {:?} at {}: {e:?}", + self.id, + self.path.display(), + ); + } + + self.file_accessor + .as_descriptor_table() + .inspect(|d| d.remove_for_blob_file(&self.global_id())); + } + } +} + +/// A blob file stores large values and is part of the value log +#[derive(Clone)] +pub struct BlobFile(pub(crate) Arc); + +impl Eq for BlobFile {} + +impl PartialEq for BlobFile { + fn eq(&self, other: &Self) -> bool { + self.id().eq(&other.id()) + } +} + +impl std::hash::Hash for BlobFile { + fn hash(&self, state: &mut H) { + self.id().hash(state); + } +} + +impl BlobFile { + pub(crate) fn mark_as_deleted(&self) { + self.0 + .is_deleted + .store(true, std::sync::atomic::Ordering::Release); + } + + /// Returns the blob file ID. + #[must_use] + pub fn id(&self) -> BlobFileId { + self.0.id + } + + /// Returns the full blob file checksum. + #[must_use] + pub fn checksum(&self) -> Checksum { + self.0.checksum + } + + /// Returns the blob file path. + #[must_use] + pub fn path(&self) -> &Path { + &self.0.path + } + + /// Returns the blob file accessor. + #[must_use] + pub(crate) fn file_accessor(&self) -> &FileAccessor { + &self.0.file_accessor + } + + /// Returns the number of items in the blob file. + #[must_use] + #[expect(clippy::len_without_is_empty)] + pub fn len(&self) -> u64 { + self.0.meta.item_count + } + + /// Returns `true` if the blob file is stale (based on the given staleness threshold). + pub(crate) fn is_stale(&self, frag_map: &FragmentationMap, threshold: f32) -> bool { + frag_map.get(&self.id()).is_some_and(|x| { + #[expect( + clippy::cast_precision_loss, + reason = "ok to lose precision as this is an approximate calculation" + )] + let stale_bytes = x.bytes as f32; + #[expect( + clippy::cast_precision_loss, + reason = "ok to lose precision as this is an approximate calculation" + )] + let all_bytes = self.0.meta.total_uncompressed_bytes as f32; + let ratio = stale_bytes / all_bytes; + ratio >= threshold + }) + } + + /// Returns `true` if the blob file has no more incoming references, and can be safely removed from a Version. + pub(crate) fn is_dead(&self, frag_map: &FragmentationMap) -> bool { + frag_map.get(&self.id()).is_some_and(|x| { + let stale_bytes = x.bytes; + let all_bytes = self.0.meta.total_uncompressed_bytes; + stale_bytes == all_bytes + }) + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/multi_writer.rs b/crates/lsm-tree/src/vlog/blob_file/multi_writer.rs new file mode 100644 index 000000000..040a7a382 --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/multi_writer.rs @@ -0,0 +1,242 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::writer::Writer; +use crate::{ + file_accessor::FileAccessor, + vlog::{ + blob_file::{Inner as BlobFileInner, Metadata}, + ValueHandle, + }, + BlobFile, CompressionType, DescriptorTable, SeqNo, SequenceNumberCounter, TreeId, +}; +use std::{ + fs::File, + path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc}, +}; + +/// Blob file writer, may write multiple blob files +pub struct MultiWriter { + folder: PathBuf, + target_size: u64, + + active_writer: Writer, + + results: Vec, + + id_generator: SequenceNumberCounter, + + compression: CompressionType, + passthrough_compression: CompressionType, + + tree_id: TreeId, + descriptor_table: Option>, +} + +impl MultiWriter { + /// Initializes a new blob file writer. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn new>( + id_generator: SequenceNumberCounter, + folder: P, + tree_id: TreeId, + descriptor_table: Option>, + ) -> crate::Result { + let folder = folder.as_ref(); + + let blob_file_id = id_generator.next(); + let blob_file_path = folder.join(blob_file_id.to_string()); + + Ok(Self { + id_generator, + folder: folder.into(), + target_size: 64 * 1_024 * 1_024, + + active_writer: Writer::new(blob_file_path, blob_file_id, tree_id)?, + + results: Vec::new(), + + compression: CompressionType::None, + passthrough_compression: CompressionType::None, + + tree_id, + descriptor_table, + }) + } + + /// Sets the blob file target size. + #[must_use] + pub fn use_target_size(mut self, bytes: u64) -> Self { + self.target_size = bytes; + self + } + + /// Sets the compression method in blob file writer metadata, but does not actually compress blobs. + /// + /// This is used in garbage collection to pass through already-compressed blobs, but correctly + /// set the compression type in the metadata. + pub(crate) fn use_passthrough_compression(mut self, compression: CompressionType) -> Self { + assert_eq!(self.compression, CompressionType::None); + self.passthrough_compression = compression; + self.active_writer.passthrough_compression = compression; + self + } + + /// Sets the compression method. + #[must_use] + #[doc(hidden)] + pub fn use_compression(mut self, compression: CompressionType) -> Self { + self.compression.clone_from(&compression); + self.active_writer.compression = compression; + self + } + + /// Sets up a new writer for the next blob file. + fn rotate(&mut self) -> crate::Result<()> { + log::debug!("Rotating blob file writer"); + + let new_blob_file_id = self.id_generator.next(); + let blob_file_path = self.folder.join(new_blob_file_id.to_string()); + + let new_writer = Writer::new(blob_file_path, new_blob_file_id, self.tree_id)? + .use_compression(self.compression) + .use_passthrough_compression(self.passthrough_compression); + + let old_writer = std::mem::replace(&mut self.active_writer, new_writer); + let blob_file = Self::consume_writer(old_writer, self.descriptor_table.clone())?; + self.results.extend(blob_file); + + Ok(()) + } + + fn consume_writer( + writer: Writer, + descriptor_table: Option>, + ) -> crate::Result> { + if writer.item_count > 0 { + let blob_file_id = writer.blob_file_id; + let path = writer.path.clone(); + + log::debug!( + "Created blob file #{blob_file_id:?} ({} items, {} userdata bytes)", + writer.item_count, + writer.uncompressed_bytes, + ); + + let tree_id = writer.tree_id; + + let (metadata, checksum) = writer.finish()?; + + let file = Arc::new(File::open(&path)?); + let file_accessor = descriptor_table.map_or(FileAccessor::File(file.clone()), |dt| { + FileAccessor::DescriptorTable(dt) + }); + file_accessor.insert_for_blob_file((tree_id, blob_file_id).into(), file); + + let blob_file = BlobFile(Arc::new(BlobFileInner { + checksum, + tree_id, + path, + is_deleted: AtomicBool::new(false), + id: blob_file_id, + file_accessor, + meta: Metadata { + id: blob_file_id, + created_at: crate::time::unix_timestamp().as_nanos(), + item_count: metadata.item_count, + total_compressed_bytes: metadata.total_compressed_bytes, + total_uncompressed_bytes: metadata.total_uncompressed_bytes, + key_range: metadata.key_range.clone(), + compression: metadata.compression, + }, + })); + + Ok(Some(blob_file)) + } else { + log::debug!( + "Blob file writer at {} has written no data, deleting empty blob file", + writer.path.display(), + ); + + if let Err(e) = std::fs::remove_file(&writer.path) { + log::warn!( + "Could not delete empty blob file at {}: {e:?}", + writer.path.display(), + ); + } + + Ok(None) + } + } + + /// Writes an item. + /// + /// Returns the [`ValueHandle`] of the written blob. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn write(&mut self, key: &[u8], seqno: SeqNo, value: &[u8]) -> crate::Result { + let target_size = self.target_size; + + // Write actual value into blob file + let writer = &mut self.active_writer; + + let offset = writer.offset(); + let on_disk_value_len = writer.write(key, seqno, value)?; + + let handle = ValueHandle { + blob_file_id: writer.blob_file_id(), + offset, + on_disk_size: on_disk_value_len, + }; + + // Check for blob file size target, maybe rotate to next writer + if writer.offset() >= target_size { + self.rotate()?; + } + + Ok(handle) + } + + pub(crate) fn write_raw( + &mut self, + key: &[u8], + seqno: SeqNo, + value: &[u8], + uncompressed_len: u32, + ) -> crate::Result { + let target_size = self.target_size; + + // Write actual value into blob file + let writer = &mut self.active_writer; + + let offset = writer.offset(); + let on_disk_value_len = writer.write_raw(key, seqno, value, uncompressed_len)?; + + let handle = ValueHandle { + blob_file_id: writer.blob_file_id(), + offset, + on_disk_size: on_disk_value_len, + }; + + // Check for blob file size target, maybe rotate to next writer + if writer.offset() >= target_size { + self.rotate()?; + } + + Ok(handle) + } + + pub(crate) fn finish(mut self) -> crate::Result> { + let blob_file = Self::consume_writer(self.active_writer, self.descriptor_table.clone())?; + self.results.extend(blob_file); + Ok(self.results) + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/reader.rs b/crates/lsm-tree/src/vlog/blob_file/reader.rs new file mode 100644 index 000000000..281b8625b --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/reader.rs @@ -0,0 +1,161 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + vlog::{ + blob_file::writer::{BLOB_HEADER_LEN, BLOB_HEADER_MAGIC}, + ValueHandle, + }, + BlobFile, Checksum, CompressionType, UserValue, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::{ + fs::File, + io::{Cursor, Read}, +}; + +/// Reads a single blob from a blob file +pub struct Reader<'a> { + blob_file: &'a BlobFile, + file: &'a File, +} + +impl<'a> Reader<'a> { + pub fn new(blob_file: &'a BlobFile, file: &'a File) -> Self { + Self { blob_file, file } + } + + pub fn get(&self, key: &'a [u8], vhandle: &'a ValueHandle) -> crate::Result { + debug_assert_eq!(vhandle.blob_file_id, self.blob_file.id()); + + let add_size = (BLOB_HEADER_LEN as u64) + (key.len() as u64); + + let value = crate::file::read_exact( + self.file, + vhandle.offset, + (u64::from(vhandle.on_disk_size) + add_size) as usize, + )?; + + let mut reader = Cursor::new(&value[..]); + + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + + if magic != BLOB_HEADER_MAGIC { + return Err(crate::Error::InvalidHeader("Blob")); + } + + let expected_checksum = reader.read_u128::()?; + + let _seqno = reader.read_u64::()?; + let key_len = reader.read_u16::()?; + + let real_val_len = reader.read_u32::()?; + + let _on_disk_val_len = reader.read_u32::()? as usize; + + let key = crate::UserKey::from_reader(&mut reader, key_len.into())?; + + let raw_data = value.slice((add_size as usize)..); + + { + let checksum = { + let mut hasher = xxhash_rust::xxh3::Xxh3::default(); + hasher.update(&key); + hasher.update(&raw_data); + hasher.digest128() + }; + + if expected_checksum != checksum { + log::error!( + "Checksum mismatch for blob {vhandle:?}, got={checksum}, expected={expected_checksum}", + ); + + return Err(crate::Error::ChecksumMismatch { + got: Checksum::from_raw(checksum), + expected: Checksum::from_raw(expected_checksum), + }); + } + } + + #[warn(clippy::match_single_binding)] + let value = match &self.blob_file.0.meta.compression { + CompressionType::None => raw_data, + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => { + #[warn(unsafe_code)] + let mut builder = unsafe { UserValue::builder_unzeroed(real_val_len as usize) }; + + lz4_flex::decompress_into(&raw_data, &mut builder) + .map_err(|_| crate::Error::Decompress(self.blob_file.0.meta.compression))?; + + builder.freeze().into() + } + }; + + debug_assert_eq!(real_val_len, { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + { + value.len() as u32 + } + }); + + Ok(value) + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::SequenceNumberCounter; + use test_log::test; + + #[test] + fn blob_reader_roundtrip() -> crate::Result<()> { + let id_generator = SequenceNumberCounter::default(); + + let folder = tempfile::tempdir()?; + let mut writer = crate::vlog::BlobFileWriter::new(id_generator, folder.path(), 0, None)? + .use_target_size(u64::MAX); + + let handle = writer.write(b"a", 0, b"abcdef")?; + + let blob_file = writer.finish()?; + let blob_file = blob_file.first().unwrap(); + + let file = File::open(&blob_file.0.path)?; + let reader = Reader::new(blob_file, &file); + + assert_eq!(reader.get(b"a", &handle)?, b"abcdef"); + + Ok(()) + } + + #[test] + #[cfg(feature = "lz4")] + fn blob_reader_roundtrip_lz4() -> crate::Result<()> { + let id_generator = SequenceNumberCounter::default(); + + let folder = tempfile::tempdir()?; + let mut writer = crate::vlog::BlobFileWriter::new(id_generator, folder.path(), 0, None)? + .use_target_size(u64::MAX) + .use_compression(CompressionType::Lz4); + + let handle0 = writer.write(b"a", 0, b"abcdef")?; + let handle1 = writer.write(b"b", 0, b"ghi")?; + + let blob_file = writer.finish()?; + let blob_file = blob_file.first().unwrap(); + + let file = File::open(&blob_file.0.path)?; + let reader = Reader::new(blob_file, &file); + + assert_eq!(reader.get(b"a", &handle0)?, b"abcdef"); + assert_eq!(reader.get(b"b", &handle1)?, b"ghi"); + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/scanner.rs b/crates/lsm-tree/src/vlog/blob_file/scanner.rs new file mode 100644 index 000000000..a4deb5def --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/scanner.rs @@ -0,0 +1,169 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::writer::BLOB_HEADER_MAGIC; +use crate::{ + vlog::{blob_file::meta::METADATA_HEADER_MAGIC, BlobFileId}, + Checksum, SeqNo, UserKey, UserValue, +}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::{ + fs::File, + io::{BufReader, Read, Seek}, + path::Path, +}; + +/// Reads through a blob file in order +pub struct Scanner { + pub(crate) blob_file_id: BlobFileId, // TODO: remove unused? + inner: BufReader, + is_terminated: bool, +} + +impl Scanner { + /// Initializes a new blob file reader. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + pub fn new>(path: P, blob_file_id: BlobFileId) -> crate::Result { + let file_reader = BufReader::with_capacity(32_000, File::open(path)?); + Ok(Self::with_reader(blob_file_id, file_reader)) + } + + /// Initializes a new blob file reader. + #[must_use] + pub fn with_reader(blob_file_id: BlobFileId, file_reader: BufReader) -> Self { + Self { + blob_file_id, + inner: file_reader, + is_terminated: false, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ScanEntry { + pub key: UserKey, + pub seqno: SeqNo, + pub value: UserValue, + pub offset: u64, + pub uncompressed_len: u32, +} + +impl Iterator for Scanner { + type Item = crate::Result; + + fn next(&mut self) -> Option { + if self.is_terminated { + return None; + } + + let offset = fail_iter!(self.inner.stream_position()); + + { + let mut buf = [0; BLOB_HEADER_MAGIC.len()]; + fail_iter!(self.inner.read_exact(&mut buf)); + + if buf == METADATA_HEADER_MAGIC { + self.is_terminated = true; + return None; + } + + if buf != BLOB_HEADER_MAGIC { + return Some(Err(crate::Error::InvalidHeader("Blob"))); + } + } + + let expected_checksum = fail_iter!(self.inner.read_u128::()); + let seqno = fail_iter!(self.inner.read_u64::()); + + let key_len = fail_iter!(self.inner.read_u16::()); + + let real_val_len = fail_iter!(self.inner.read_u32::()); + + let on_disk_val_len = fail_iter!(self.inner.read_u32::()); + + let key = fail_iter!(UserKey::from_reader(&mut self.inner, key_len as usize)); + + let value = fail_iter!(UserValue::from_reader( + &mut self.inner, + on_disk_val_len as usize + )); + + { + let checksum = { + let mut hasher = xxhash_rust::xxh3::Xxh3::default(); + hasher.update(&key); + hasher.update(&value); + hasher.digest128() + }; + + if expected_checksum != checksum { + log::error!( + "Checksum mismatch for blob>{}@{offset}, got={checksum}, expected={expected_checksum}", + self.blob_file_id, + ); + + return Some(Err(crate::Error::ChecksumMismatch { + got: Checksum::from_raw(checksum), + expected: Checksum::from_raw(expected_checksum), + })); + } + } + + Some(Ok(ScanEntry { + key, + seqno, + value, + offset, + uncompressed_len: real_val_len, + })) + } +} + +#[cfg(test)] +#[expect(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::{vlog::blob_file::writer::Writer as BlobFileWriter, Slice}; + use tempfile::tempdir; + use test_log::test; + + #[test] + fn blob_scanner() -> crate::Result<()> { + let dir = tempdir()?; + let blob_file_path = dir.path().join("0"); + + let keys = [b"a", b"b", b"c", b"d", b"e"]; + + { + let mut writer = BlobFileWriter::new(&blob_file_path, 0, 0)?; + + for key in keys { + writer.write(key, 0, &key.repeat(100))?; + } + + writer.finish()?; + } + + { + let mut scanner = Scanner::new(&blob_file_path, 0)?; + + for key in keys { + assert_eq!( + (Slice::from(key), Slice::from(key.repeat(100))), + scanner + .next() + .map(|result| result.map(|entry| { (entry.key, entry.value) })) + .unwrap()?, + ); + } + + assert!(scanner.next().is_none()); + } + + Ok(()) + } +} diff --git a/crates/lsm-tree/src/vlog/blob_file/writer.rs b/crates/lsm-tree/src/vlog/blob_file/writer.rs new file mode 100644 index 000000000..77298c451 --- /dev/null +++ b/crates/lsm-tree/src/vlog/blob_file/writer.rs @@ -0,0 +1,247 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use super::meta::Metadata; +use crate::{ + checksum::ChecksummedWriter, time::unix_timestamp, vlog::BlobFileId, Checksum, CompressionType, + KeyRange, SeqNo, TreeId, UserKey, +}; +use byteorder::{LittleEndian, WriteBytesExt}; +use std::{ + fs::File, + io::{BufWriter, Write}, + path::{Path, PathBuf}, +}; + +pub const BLOB_HEADER_MAGIC: &[u8] = b"BLOB"; + +pub const BLOB_HEADER_LEN: usize = BLOB_HEADER_MAGIC.len() + + std::mem::size_of::() // Checksum + + std::mem::size_of::() // SeqNo + + std::mem::size_of::() // Key length + + std::mem::size_of::() // Real value length + + std::mem::size_of::(); // On-disk value length + +/// Blob file writer +pub struct Writer { + pub(crate) tree_id: TreeId, + pub path: PathBuf, + pub(crate) blob_file_id: BlobFileId, + + #[expect(clippy::struct_field_names)] + writer: sfa::Writer>>, + + offset: u64, + + pub(crate) item_count: u64, + pub(crate) written_blob_bytes: u64, + pub(crate) uncompressed_bytes: u64, + + pub(crate) first_key: Option, + pub(crate) last_key: Option, + + pub(crate) compression: CompressionType, + pub(crate) passthrough_compression: CompressionType, +} + +impl Writer { + /// Initializes a new blob file writer. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + #[doc(hidden)] + pub fn new>( + path: P, + blob_file_id: BlobFileId, + tree_id: TreeId, + ) -> crate::Result { + let path = path.as_ref(); + + let writer = BufWriter::new(File::create(path)?); + let writer = ChecksummedWriter::new(writer); + let mut writer = sfa::Writer::from_writer(writer); + writer.start("data")?; + + Ok(Self { + tree_id, + path: path.into(), + blob_file_id, + + writer, + + offset: 0, + item_count: 0, + written_blob_bytes: 0, + uncompressed_bytes: 0, + + first_key: None, + last_key: None, + + compression: CompressionType::None, + passthrough_compression: CompressionType::None, + }) + } + + pub fn use_compression(mut self, compressor: CompressionType) -> Self { + self.compression = compressor; + self + } + + pub(crate) fn use_passthrough_compression(mut self, compressor: CompressionType) -> Self { + self.passthrough_compression = compressor; + self + } + + /// Returns the current offset in the file. + #[must_use] + pub(crate) fn offset(&self) -> u64 { + self.offset + } + + /// Returns the blob file ID. + #[must_use] + pub(crate) fn blob_file_id(&self) -> BlobFileId { + self.blob_file_id + } + + pub(crate) fn write_raw( + &mut self, + key: &[u8], + seqno: SeqNo, + value: &[u8], + uncompressed_len: u32, + ) -> crate::Result { + assert!(!key.is_empty()); + assert!(u16::try_from(key.len()).is_ok()); + assert!(u32::try_from(value.len()).is_ok()); + + if self.first_key.is_none() { + self.first_key = Some(key.into()); + } + self.last_key = Some(key.into()); + + self.uncompressed_bytes += u64::from(uncompressed_len); + + // NOTE: + // BLOB HEADER LAYOUT + // + // [MAGIC_BYTES; 4B] + // [Checksum; 16B] + // [Seqno; 8B] + // [key len; 2B] + // [real val len; 4B] + // [on-disk val len; 4B] + // [...key; ?] + // [...val; ?] + + // Write header + self.writer.write_all(BLOB_HEADER_MAGIC)?; + + let value = match &self.compression { + CompressionType::None => std::borrow::Cow::Borrowed(value), + + #[cfg(feature = "lz4")] + CompressionType::Lz4 => std::borrow::Cow::Owned(lz4_flex::compress(value)), + }; + + let checksum = { + let mut hasher = xxhash_rust::xxh3::Xxh3::default(); + hasher.update(key); + hasher.update(&value); + hasher.digest128() + }; + + // Write checksum + self.writer.write_u128::(checksum)?; + + // Write seqno + self.writer.write_u64::(seqno)?; + + #[expect(clippy::cast_possible_truncation, reason = "keys are u16 length max")] + self.writer.write_u16::(key.len() as u16)?; + + // Write uncompressed value length + self.writer.write_u32::(uncompressed_len)?; + + // Write compressed (on-disk) value length + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + self.writer.write_u32::(value.len() as u32)?; + + self.writer.write_all(key)?; + self.writer.write_all(&value)?; + + // Update offset + self.offset += BLOB_HEADER_MAGIC.len() as u64; + self.offset += std::mem::size_of::() as u64; + self.offset += std::mem::size_of::() as u64; + + self.offset += std::mem::size_of::() as u64; + self.offset += std::mem::size_of::() as u64; + self.offset += std::mem::size_of::() as u64; + + self.offset += key.len() as u64; + self.offset += value.len() as u64; + + // Update metadata + self.written_blob_bytes += value.len() as u64; + self.item_count += 1; + + // TODO: if we store the offset before writing, we can return a vhandle here + // instead of needing to call offset() and blob_file_id() before write() + + #[expect(clippy::cast_possible_truncation, reason = "values are u32 length max")] + Ok(value.len() as u32) + } + + /// Writes an item into the file. + /// + /// Items need to be written in key order. + /// + /// # Errors + /// + /// Will return `Err` if an IO error occurs. + /// + /// # Panics + /// + /// Panics if the key length is empty or greater than 2^16, or the value length is greater than 2^32. + pub fn write(&mut self, key: &[u8], seqno: SeqNo, value: &[u8]) -> crate::Result { + #[expect(clippy::cast_possible_truncation, reason = "values are u32 max")] + self.write_raw(key, seqno, value, value.len() as u32) + } + + pub(crate) fn finish(mut self) -> crate::Result<(Metadata, Checksum)> { + self.writer.start("meta")?; + + // Write metadata + let metadata = Metadata { + id: self.blob_file_id, + created_at: unix_timestamp().as_nanos(), + item_count: self.item_count, + total_compressed_bytes: self.written_blob_bytes, + total_uncompressed_bytes: self.uncompressed_bytes, + #[expect(clippy::expect_used, reason = "should have written at least 1 item")] + key_range: KeyRange::new(( + self.first_key + .clone() + .expect("should have written at least 1 item"), + self.last_key + .clone() + .expect("should have written at least 1 item"), + )), + compression: if self.passthrough_compression == CompressionType::None { + self.compression + } else { + self.passthrough_compression + }, + }; + metadata.encode_into(&mut self.writer)?; + + let mut checksum = self.writer.into_inner()?; + checksum.inner_mut().get_mut().sync_all()?; + let checksum = checksum.checksum(); + + Ok((metadata, checksum)) + } +} diff --git a/crates/lsm-tree/src/vlog/handle.rs b/crates/lsm-tree/src/vlog/handle.rs new file mode 100644 index 000000000..d0888147b --- /dev/null +++ b/crates/lsm-tree/src/vlog/handle.rs @@ -0,0 +1,49 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use crate::{ + coding::{Decode, Encode}, + vlog::BlobFileId, +}; +use std::{ + hash::Hash, + io::{Read, Write}, +}; +use varint_rs::{VarintReader, VarintWriter}; + +/// A value handle points into the value log +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +pub struct ValueHandle { + /// Blob file ID + pub blob_file_id: BlobFileId, + + /// Offset in file + pub offset: u64, + + /// On-disk size + pub on_disk_size: u32, +} + +impl Encode for ValueHandle { + fn encode_into(&self, writer: &mut W) -> Result<(), crate::Error> { + writer.write_u64_varint(self.offset)?; + writer.write_u64_varint(self.blob_file_id)?; + writer.write_u32_varint(self.on_disk_size)?; + Ok(()) + } +} + +impl Decode for ValueHandle { + fn decode_from(reader: &mut R) -> Result { + let offset = reader.read_u64_varint()?; + let blob_file_id = reader.read_u64_varint()?; + let on_disk_size = reader.read_u32_varint()?; + + Ok(Self { + blob_file_id, + offset, + on_disk_size, + }) + } +} diff --git a/crates/lsm-tree/src/vlog/mod.rs b/crates/lsm-tree/src/vlog/mod.rs new file mode 100644 index 000000000..5f59a0a7d --- /dev/null +++ b/crates/lsm-tree/src/vlog/mod.rs @@ -0,0 +1,150 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +mod accessor; +pub mod blob_file; +mod handle; + +pub use { + accessor::Accessor, blob_file::merge::MergeScanner as BlobFileMergeScanner, + blob_file::multi_writer::MultiWriter as BlobFileWriter, + blob_file::scanner::Scanner as BlobFileScanner, blob_file::BlobFile, handle::ValueHandle, +}; + +use crate::{ + file_accessor::FileAccessor, + vlog::blob_file::{Inner as BlobFileInner, Metadata}, + Checksum, DescriptorTable, TreeId, +}; +use std::{ + path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc}, +}; + +pub fn recover_blob_files( + folder: &Path, + ids: &[(BlobFileId, Checksum)], + tree_id: TreeId, + descriptor_table: Option<&Arc>, +) -> crate::Result<(Vec, Vec)> { + if !folder.try_exists()? { + return Ok((vec![], vec![])); + } + + let cnt = ids.len(); + + let progress_mod = match cnt { + _ if cnt <= 20 => 1, + _ if cnt <= 100 => 10, + _ => 100, + }; + + log::debug!("Recovering {cnt} blob files from {:?}", folder.display()); + + let mut blob_files = Vec::with_capacity(ids.len()); + let mut orphaned_blob_files = vec![]; + + for (idx, dirent) in std::fs::read_dir(folder)?.enumerate() { + let dirent = dirent?; + let file_name = dirent.file_name(); + + // https://en.wikipedia.org/wiki/.DS_Store + if file_name == ".DS_Store" { + continue; + } + + // https://en.wikipedia.org/wiki/AppleSingle_and_AppleDouble_formats + if file_name.to_string_lossy().starts_with("._") { + continue; + } + + let blob_file_name = file_name.to_str().ok_or_else(|| { + log::error!("invalid table file name {}", file_name.display()); + crate::Error::Unrecoverable + })?; + + let blob_file_id = blob_file_name.parse::().map_err(|e| { + log::error!("invalid table file name {blob_file_name:?}: {e:?}"); + crate::Error::Unrecoverable + })?; + + let blob_file_path = dirent.path(); + assert!(!blob_file_path.is_dir()); + + if let Some(&(_, checksum)) = ids.iter().find(|(id, _)| id == &blob_file_id) { + log::trace!( + "Recovering blob file #{blob_file_id:?} from {}", + blob_file_path.display(), + ); + + let file = std::fs::File::open(&blob_file_path)?; + + let meta = { + let reader = sfa::Reader::new(&blob_file_path)?; + let toc = reader.toc(); + + let metadata_section = toc.section(b"meta") + .ok_or(crate::Error::Unrecoverable) + .inspect_err(|_| { + log::error!("meta section in blob file #{blob_file_id} is missing - maybe the file is corrupted?"); + })?; + + let metadata_slice = crate::file::read_exact( + &file, + metadata_section.pos(), + metadata_section.len() as usize, + )?; + + Metadata::from_slice(&metadata_slice)? + }; + + let file_accessor = if let Some(dt) = descriptor_table.cloned() { + FileAccessor::DescriptorTable(dt) + } else { + FileAccessor::File(Arc::new(file)) + }; + + blob_files.push(BlobFile(Arc::new(BlobFileInner { + id: blob_file_id, + path: blob_file_path, + meta, + is_deleted: AtomicBool::new(false), + checksum, + file_accessor, + tree_id, + }))); + + if idx % progress_mod == 0 { + log::debug!("Recovered {idx}/{cnt} blob files"); + } + } else { + orphaned_blob_files.push(blob_file_path.clone()); + } + } + + if blob_files.len() < ids.len() { + return Err(crate::Error::Unrecoverable); + } + + log::debug!("Successfully recovered {} blob files", blob_files.len()); + + Ok((blob_files, orphaned_blob_files)) +} + +/// The unique identifier for a value log blob file. +pub type BlobFileId = u64; + +#[cfg(test)] +mod tests { + use super::*; + use test_log::test; + + #[test] + fn vlog_recovery_missing_blob_file() { + assert!(matches!( + recover_blob_files(Path::new("."), &[(0, Checksum::from_raw(0))], 0, None), + Err(crate::Error::Unrecoverable), + )); + } +} diff --git a/crates/lsm-tree/test_fixture/v1_tree/config b/crates/lsm-tree/test_fixture/v1_tree/config new file mode 100644 index 0000000000000000000000000000000000000000..61e5bd120bd4eff9fc05bda9d16a304c1be34a9d GIT binary patch literal 16 XcmZ?s^6_zYb2nsQWME(rU|HlGEx;%D^rVg6!P**faWI_W#*|f*fBIRrT}$m z1(%331OnBS}pPJUtv(C*A)g_OjS#H7UHRC^PK z4~iK;wb~gaB?ZM+`ugSN<$6F@mXs!?>Lusr>X#>$Bxl%{*``JYn^)v!1(~>IyTO7{ zjAi;_pk5dd*gaXD4xN6R=}~Lzuz^P=?_3X`=F` z{?n)238~mR=i5C!?K|`SgNzRLb%85}X<}fLSn=wj{A?(n1C53&gD`;hf&o(v1A_pF n#fWe|LHlGEx;%D^rVg6!P**faWI_W#*|f*fBIRrT}$m z1(%331OnBS}pPJUtv(C*A)g_OjS#H7TiRC^PK z4~iK;wb~gaB?ZM+`ugSN<$6F@mXs!?>Lusr>X#>$Bxl%{*``JYn^)v!1(~>IyTO7{ zjAi;_pk5dd*gaXD4xN6R=}~Lzuz^P=?_3X`=F` z{?n)238~mR=i5C!?K|`SgNzRLb%85}X<}fLSn=wj{A?(n1C53&gD`;hf&o(v1A_pF n#fWe|L lsm_tree::Result<()> { + use lsm_tree::{ + blob_tree::FragmentationEntry, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, + }; + + let folder = tempfile::tempdir()?; + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .compression(lsm_tree::CompressionType::Lz4) + .separation_threshold(1) + .staleness_threshold(0.0000001) + .age_cutoff(1.0), + )) + .open()?; + + let big_value = b"abc".repeat(50); + + tree.insert("a", &big_value, 0); + tree.insert("b", b"smol", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + + let value = tree.get("b", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, b"smol"); + } + + tree.remove("b", 1); + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + + assert!(!tree.contains_key("b", SeqNo::MAX)?); + } + + tree.major_compact(u64::MAX, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + map.insert( + 0, + FragmentationEntry::new(1, b"smol".len().try_into().unwrap(), 5), + ); + map + }, + &*gc_stats, + ); + } + + { + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + + assert!(!tree.contains_key("b", SeqNo::MAX)?); + } + + tree.major_compact(u64::MAX, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + { + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + + assert!(!tree.contains_key("b", SeqNo::MAX)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_empty_flush.rs b/crates/lsm-tree/tests/blob_empty_flush.rs new file mode 100644 index 000000000..56a72b053 --- /dev/null +++ b/crates/lsm-tree/tests/blob_empty_flush.rs @@ -0,0 +1,30 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_flush_empty() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let medium_value = b"a".repeat(500); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("med", &medium_value, 0); + tree.flush_active_memtable(0)?; + + assert_eq!(1, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + + // Blob writer should have cleaned up blob file because it was empty + let blob_file_count_on_disk = std::fs::read_dir(path.join("blobs"))?.flatten().count(); + assert_eq!(0, blob_file_count_on_disk); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_fifo_limit.rs b/crates/lsm-tree/tests/blob_fifo_limit.rs new file mode 100644 index 000000000..00ae4db59 --- /dev/null +++ b/crates/lsm-tree/tests/blob_fifo_limit.rs @@ -0,0 +1,28 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn blob_tree_fifo_limit() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let compaction = Arc::new(lsm_tree::compaction::Fifo::new(10, None)); + + for _ in 0..100 { + tree.insert(nanoid::nanoid!(), "$", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert!((0..10).contains(&tree.blob_file_count())); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_file_full_file_checksum.rs b/crates/lsm-tree/tests/blob_file_full_file_checksum.rs new file mode 100644 index 000000000..d5cb8c07f --- /dev/null +++ b/crates/lsm-tree/tests/blob_file_full_file_checksum.rs @@ -0,0 +1,121 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter}; +use test_log::test; +use xxhash_rust::xxh3::xxh3_128; + +#[test] +fn blob_file_full_file_checksum() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + for key in ('a'..='z').map(|c| c.to_string()) { + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + let version = tree.current_version(); + let blob_file = version.blob_files.iter().next().unwrap(); + + let expected_checksum = blob_file.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(blob_file.path())?); + assert_eq!( + real_checksum, expected_checksum, + "full file checksum mismatch", + ); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let version = tree.current_version(); + let blob_file = version.blob_files.iter().next().unwrap(); + + let expected_checksum = blob_file.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(blob_file.path())?); + assert_eq!( + real_checksum, expected_checksum, + "full file checksum mismatch", + ); + } + + Ok(()) +} + +#[test] +fn blob_file_full_file_detect_corruption() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + for key in ('a'..='z').map(|c| c.to_string()) { + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + let version = tree.current_version(); + let blob_file = version.blob_files.iter().next().unwrap(); + + let expected_checksum = blob_file.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(blob_file.path())?); + assert_eq!( + real_checksum, expected_checksum, + "full file checksum mismatch", + ); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let version = tree.current_version(); + let blob_file = version.blob_files.iter().next().unwrap(); + + { + use std::io::{Seek, Write}; + + let mut f = std::fs::OpenOptions::new() + .write(true) + .open(blob_file.path())?; + + f.seek(std::io::SeekFrom::Start(100))?; + f.write_all(b"!")?; + f.sync_all()?; + } + + let expected_checksum = blob_file.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(blob_file.path())?); + assert_ne!( + real_checksum, expected_checksum, + "full file checksum should not match", + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_flush_gc_stats.rs b/crates/lsm-tree/tests/blob_flush_gc_stats.rs new file mode 100644 index 000000000..c88489c79 --- /dev/null +++ b/crates/lsm-tree/tests/blob_flush_gc_stats.rs @@ -0,0 +1,71 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_flush_gc_stats() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.insert("big", &new_big_value, 1); + + tree.flush_active_memtable(1_000)?; + + // NOTE: The first big_value is dropped, so it never arrives in a blob file + assert_eq!(2, tree.approximate_len()); + } + + Ok(()) +} + +#[test] +fn blob_tree_flush_gc_stats_tombstone() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.remove("big", 1); + + tree.flush_active_memtable(1_000)?; + + // NOTE: The first big_value is dropped, so it never arrives in a blob file + assert_eq!(2, tree.approximate_len()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_gc_watermark.rs b/crates/lsm-tree/tests/blob_gc_watermark.rs new file mode 100644 index 000000000..5cb2c0282 --- /dev/null +++ b/crates/lsm-tree/tests/blob_gc_watermark.rs @@ -0,0 +1,105 @@ +use lsm_tree::{ + config::CompressionPolicy, get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +// NOTE: This was a logic/MVCC error in v2 that could drop +// a blob file while it was maybe accessible by a snapshot read +// +// https://github.com/fjall-rs/lsm-tree/commit/79c6ead4b955051cbb4835913e21d08b8aeafba1 +#[test] +fn blob_gc_seqno_watermark() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_compression_policy(CompressionPolicy::all(lsm_tree::CompressionType::None)) + .with_kv_separation(Some( + KvSeparationOptions::default() + .staleness_threshold(0.01) + .age_cutoff(1.0) + .separation_threshold(50), + )) + .open()?; + + tree.insert("a", "neptune".repeat(50), seqno.next()); + + let snapshot_seqno = seqno.get(); + + assert_eq!( + &*tree.get("a", snapshot_seqno)?.unwrap(), + b"neptune".repeat(50), + ); + assert_eq!(&*tree.get("a", SeqNo::MAX)?.unwrap(), b"neptune".repeat(50),); + + tree.insert("a", "neptune2".repeat(50), seqno.next()); + assert_eq!( + &*tree.get("a", snapshot_seqno)?.unwrap(), + b"neptune".repeat(50), + ); + assert_eq!( + &*tree.get("a", SeqNo::MAX)?.unwrap(), + b"neptune2".repeat(50), + ); + + tree.insert("a", "neptune3".repeat(50), seqno.next()); + assert_eq!( + &*tree.get("a", snapshot_seqno)?.unwrap(), + b"neptune".repeat(50), + ); + assert_eq!( + &*tree.get("a", SeqNo::MAX)?.unwrap(), + b"neptune3".repeat(50), + ); + + tree.flush_active_memtable(0)?; + + assert_eq!( + &*tree.get("a", snapshot_seqno)?.unwrap(), + b"neptune".repeat(50), + ); + assert_eq!( + &*tree.get("a", SeqNo::MAX)?.unwrap(), + b"neptune3".repeat(50), + ); + + tree.major_compact(u64::MAX, 0)?; + tree.major_compact(u64::MAX, 0)?; + + // IMPORTANT: We cannot drop any blobs yet + // because the watermark is too low + // + // This would previously fail + + { + let gc_stats = tree.current_version().gc_stats().clone(); + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + assert_eq!( + &*tree.get("a", snapshot_seqno)?.unwrap(), + b"neptune".repeat(50), + ); + assert_eq!( + &*tree.get("a", SeqNo::MAX)?.unwrap(), + b"neptune3".repeat(50), + ); + + tree.major_compact(u64::MAX, 1_000)?; + + { + let gc_stats = tree.current_version().gc_stats().clone(); + assert!(!gc_stats.is_empty()); + } + + tree.major_compact(u64::MAX, 1_000)?; + + { + let gc_stats = tree.current_version().gc_stats().clone(); + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_ingest.rs b/crates/lsm-tree/tests/blob_ingest.rs new file mode 100644 index 000000000..93cef156e --- /dev/null +++ b/crates/lsm-tree/tests/blob_ingest.rs @@ -0,0 +1,61 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn blob_ingest_gc_stats() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + let mut ingestion = tree.ingestion()?; + ingestion.write("big", &big_value)?; + ingestion.write("smol", "small value")?; + ingestion.finish()?; + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let mut ingestion = tree.ingestion()?; + ingestion.write("big", &new_big_value)?; + ingestion.finish()?; + + // Blob file has no fragmentation before compaction (in stats) + // so it is not rewritten + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 is expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_ingest_relink.rs b/crates/lsm-tree/tests/blob_ingest_relink.rs new file mode 100644 index 000000000..619d0f451 --- /dev/null +++ b/crates/lsm-tree/tests/blob_ingest_relink.rs @@ -0,0 +1,66 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_ingest_relink() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + let mut ingestion = tree.ingestion()?; + ingestion.write("big", &big_value)?; + ingestion.write("smol", "small value")?; + ingestion.finish()?; + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + assert_eq!( + Some(vec![lsm_tree::table::writer::LinkedFile { + blob_file_id: 0, + bytes: big_value.len() as u64, + on_disk_bytes: big_value.len() as u64, + len: 1, + }]), + tree.current_version() + .iter_tables() + .next() + .unwrap() + .list_blob_file_references()?, + ); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + assert_eq!( + Some(vec![lsm_tree::table::writer::LinkedFile { + blob_file_id: 0, + bytes: big_value.len() as u64, + on_disk_bytes: big_value.len() as u64, + len: 1, + }]), + tree.current_version() + .iter_tables() + .next() + .unwrap() + .list_blob_file_references()?, + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_major_compact_drop_dead_files.rs b/crates/lsm-tree/tests/blob_major_compact_drop_dead_files.rs new file mode 100644 index 000000000..5509fb3a6 --- /dev/null +++ b/crates/lsm-tree/tests/blob_major_compact_drop_dead_files.rs @@ -0,0 +1,100 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn blob_tree_major_compact_drop_dead_files() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + + tree.insert("big", &big_value, 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.insert("big", &big_value, 1); + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + tree.insert("big", &big_value, 2); + tree.flush_active_memtable(0)?; + assert_eq!(3, tree.table_count()); + assert_eq!(3, tree.blob_file_count()); + + tree.insert("big", &big_value, 3); + tree.flush_active_memtable(0)?; + assert_eq!(4, tree.table_count()); + assert_eq!(4, tree.blob_file_count()); + + tree.insert("big", &new_big_value, 4); + tree.flush_active_memtable(0)?; + assert_eq!(5, tree.table_count()); + assert_eq!(5, tree.blob_file_count()); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(5, tree.blob_file_count()); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map.insert(1, FragmentationEntry::new(1, size, size)); + map.insert(2, FragmentationEntry::new(1, size, size)); + map.insert(3, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_major_compact_gc_stats.rs b/crates/lsm-tree/tests/blob_major_compact_gc_stats.rs new file mode 100644 index 000000000..3f5d147f7 --- /dev/null +++ b/crates/lsm-tree/tests/blob_major_compact_gc_stats.rs @@ -0,0 +1,205 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn blob_tree_major_compact_gc_stats() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.insert("big", &new_big_value, 1); + + tree.flush_active_memtable(0)?; + + // Blob file has no fragmentation before compaction (in stats) + // so it is not rewritten + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 is expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + assert_eq!(big_value.len() as u64, tree.stale_blob_bytes()); + } + + Ok(()) +} + +#[test] +fn blob_tree_major_compact_gc_stats_2() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("big", &big_value, 1); + tree.insert("big", &big_value, 2); + tree.insert("big", &big_value, 3); + tree.insert("big", &big_value, 4); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + // Blob file has no fragmentation before compaction (in stats) + // so it is not rewritten + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0,1,2,3 are expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(4, size * 4, size * 4)); + map + }, + &*gc_stats, + ); + } + + Ok(()) +} + +#[test] +fn blob_tree_major_compact_gc_stats_tombstone() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("another_big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.remove("big", 1); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + assert_eq!( + Some(vec![lsm_tree::table::writer::LinkedFile { + blob_file_id: 0, + bytes: 2 * big_value.len() as u64, + on_disk_bytes: 2 * big_value.len() as u64, + len: 2, + }]), + tree.current_version() + .iter_tables() + .nth(1) + .unwrap() + .list_blob_file_references()?, + ); + + // Blob file has no fragmentation before compaction (in stats) + // so it is not rewritten + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 is expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + + assert_eq!( + Some(vec![lsm_tree::table::writer::LinkedFile { + blob_file_id: 0, + bytes: big_value.len() as u64, + on_disk_bytes: big_value.len() as u64, + len: 1, + }]), + tree.current_version() + .iter_tables() + .next() + .unwrap() + .list_blob_file_references()?, + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_major_compact_relink.rs b/crates/lsm-tree/tests/blob_major_compact_relink.rs new file mode 100644 index 000000000..c2bdcba7e --- /dev/null +++ b/crates/lsm-tree/tests/blob_major_compact_relink.rs @@ -0,0 +1,69 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_major_compact_relink() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + assert_eq!( + Some(vec![lsm_tree::table::writer::LinkedFile { + blob_file_id: 0, + bytes: big_value.len() as u64, + on_disk_bytes: big_value.len() as u64, + len: 1, + }]), + tree.current_version() + .iter_tables() + .next() + .unwrap() + .list_blob_file_references()?, + ); + + tree.flush_active_memtable(1)?; + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + assert_eq!( + Some(vec![lsm_tree::table::writer::LinkedFile { + blob_file_id: 0, + bytes: big_value.len() as u64, + on_disk_bytes: big_value.len() as u64, + len: 1, + }]), + tree.current_version() + .iter_tables() + .next() + .unwrap() + .list_blob_file_references()?, + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_major_compact_relocation.rs b/crates/lsm-tree/tests/blob_major_compact_relocation.rs new file mode 100644 index 000000000..e915fa8f6 --- /dev/null +++ b/crates/lsm-tree/tests/blob_major_compact_relocation.rs @@ -0,0 +1,326 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn blob_tree_major_compact_relocation_simple() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .compression(lsm_tree::CompressionType::None) + .age_cutoff(1.0), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("big2", &big_value, 0); + tree.insert("smol", "smol", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("big2", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.insert("big", &new_big_value, 1); + + tree.flush_active_memtable(0)?; + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + let value = tree.get("big2", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + let value = tree.get("big2", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 is expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + let value = tree.get("big2", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + } + + Ok(()) +} + +#[test] +fn blob_tree_major_compact_relocation_repeated_key() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(2_000); + let very_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .compression(lsm_tree::CompressionType::None) + .age_cutoff(1.0), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("a", &big_value, 0); + tree.insert("b", &big_value, 0); + tree.insert("c", &very_big_value, 0); + tree.insert("d", &big_value, 0); + tree.insert("e", &big_value, 0); + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, very_big_value); + let value = tree.get("d", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.remove("c", 1); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?; + assert!(value.is_none()); + let value = tree.get("d", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?; + assert!(value.is_none()); + let value = tree.get("d", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = very_big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?; + assert!(value.is_none()); + let value = tree.get("d", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + } + + Ok(()) +} + +#[test] +fn blob_tree_major_compact_relocation_interleaved() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(2_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .compression(lsm_tree::CompressionType::None) + .age_cutoff(1.0), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("a", b"smol", 0); + tree.insert("b", &big_value, 0); + tree.insert("c", b"smol", 0); + tree.insert("d", &big_value, 0); + tree.insert("e", b"smol", 0); + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("d", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.remove("d", 1); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("d", SeqNo::MAX)?; + assert!(value.is_none()); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("d", SeqNo::MAX)?; + assert!(value.is_none()); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + let value = tree.get("a", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("b", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + let value = tree.get("c", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + let value = tree.get("d", SeqNo::MAX)?; + assert!(value.is_none()); + let value = tree.get("e", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"smol"); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_major_compact_relocation_recovery.rs b/crates/lsm-tree/tests/blob_major_compact_relocation_recovery.rs new file mode 100644 index 000000000..fd5b536f1 --- /dev/null +++ b/crates/lsm-tree/tests/blob_major_compact_relocation_recovery.rs @@ -0,0 +1,130 @@ +#[test_log::test] +fn blob_tree_major_compact_relocation_recovery_no_comp() -> lsm_tree::Result<()> { + use lsm_tree::{ + get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter, + }; + + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(4_096); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .compression(lsm_tree::CompressionType::None) + .separation_threshold(1) + .staleness_threshold(0.0000001) + .age_cutoff(1.0), + )) + .open()?; + + tree.insert("a", &big_value, 0); + tree.insert("b", b"smol", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.remove("b", 1); + tree.flush_active_memtable(0)?; + + tree.major_compact(u64::MAX, 1_000)?; + tree.major_compact(u64::MAX, 1_000)?; + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + } + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + } + + Ok(()) +} + +#[test_log::test] +#[cfg(feature = "lz4")] +fn blob_tree_major_compact_relocation_recovery() -> lsm_tree::Result<()> { + use lsm_tree::{ + get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter, + }; + + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(4_096); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .compression(lsm_tree::CompressionType::Lz4) + .separation_threshold(1) + .staleness_threshold(0.0000001) + .age_cutoff(1.0), + )) + .open()?; + + tree.insert("a", &big_value, 0); + tree.insert("b", b"smol", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.remove("b", 1); + tree.flush_active_memtable(0)?; + + tree.major_compact(u64::MAX, 1_000)?; + tree.major_compact(u64::MAX, 1_000)?; + assert_eq!(1, tree.blob_file_count()); + + { + let gc_stats = tree.current_version().gc_stats().clone(); + + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats); + } + + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + } + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + let value = tree.get("a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*value, big_value); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_nuke_gc_stats.rs b/crates/lsm-tree/tests/blob_nuke_gc_stats.rs new file mode 100644 index 000000000..faecd5788 --- /dev/null +++ b/crates/lsm-tree/tests/blob_nuke_gc_stats.rs @@ -0,0 +1,110 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn blob_tree_nuke_gc_stats() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.drop_range::<&[u8], _>(..)?; + + // NOTE: Because the blob does not have any incoming references anymore + // it is pruned from the Version + assert_eq!(0, tree.blob_file_count()); + assert_eq!(0, tree.table_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 was dropped + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + Ok(()) +} + +#[test] +fn blob_tree_nuke_gc_stats_multi() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("big", &big_value, 1); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.drop_range::<&[u8], _>(..)?; + + // NOTE: Because the blob does not have any incoming references anymore + // it is pruned from the Version + assert_eq!(0, tree.blob_file_count()); + assert_eq!(0, tree.table_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0, big":1 were dropped + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(2, size * 2, size * 2)); + map + }, + &*gc_stats, + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_pinned_fd.rs b/crates/lsm-tree/tests/blob_pinned_fd.rs new file mode 100644 index 000000000..d793fe0cd --- /dev/null +++ b/crates/lsm-tree/tests/blob_pinned_fd.rs @@ -0,0 +1,58 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_with_pinned_fd() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .use_descriptor_table(None) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"small value"); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + } + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .use_descriptor_table(None) + .with_kv_separation(Some(Default::default())) + .open()?; + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"small value"); + + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_recover_gc_stats.rs b/crates/lsm-tree/tests/blob_recover_gc_stats.rs new file mode 100644 index 000000000..40b618569 --- /dev/null +++ b/crates/lsm-tree/tests/blob_recover_gc_stats.rs @@ -0,0 +1,82 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn blob_tree_recover_gc_stats() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.insert("big", &new_big_value, 1); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 is expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "big":0 is still expired + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_recovery.rs b/crates/lsm-tree/tests/blob_recovery.rs new file mode 100644 index 000000000..391f36446 --- /dev/null +++ b/crates/lsm-tree/tests/blob_recovery.rs @@ -0,0 +1,49 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_recovery() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(4_096); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("big", &big_value, 0); + tree.flush_active_memtable(0)?; + + tree.insert("big", &big_value, 0); + tree.flush_active_memtable(0)?; + + tree.insert("big", &big_value, 0); + tree.flush_active_memtable(0)?; + + tree.insert("big", &big_value, 0); + tree.flush_active_memtable(0)?; + + tree.insert("big", &big_value, 0); + tree.flush_active_memtable(0)?; + } + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert!(!tree.is_empty(SeqNo::MAX, None)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_register_table_rotation.rs b/crates/lsm-tree/tests/blob_register_table_rotation.rs new file mode 100644 index 000000000..64d13fb97 --- /dev/null +++ b/crates/lsm-tree/tests/blob_register_table_rotation.rs @@ -0,0 +1,135 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, KvSeparationOptions, + SequenceNumberCounter, +}; +use test_log::test; + +// Force one block per table and one blob per block +// +// Then check if item_count in a table matches the number of referenced blobs (so 1). +// +// See https://github.com/fjall-rs/lsm-tree/commit/0d2d7b2071c65f2538bb01e4512907892991dcbe +#[test] +fn blob_register_table_rotation() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1)) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(0) + .age_cutoff(1.0) + .staleness_threshold(0.0), + )) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.insert("d", "d", 0); + tree.insert("e", "e", 0); + + tree.flush_active_memtable(0)?; + tree.major_compact(1, 0)?; + + assert_eq!(5, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + for table in tree.current_version().iter_tables() { + assert_eq!( + 1, + table + .list_blob_file_references()? + .unwrap() + .iter() + .map(|x| x.len) + .sum::(), + ); + assert_eq!( + 1, + table + .list_blob_file_references()? + .unwrap() + .iter() + .map(|x| x.bytes) + .sum::(), + ); + } + + Ok(()) +} + +#[test] +fn blob_register_table_rotation_relocation() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1)) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(0) + .age_cutoff(1.0) + .staleness_threshold(0.0), + )) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.insert("d", "d", 0); + tree.insert("e", "e", 0); + tree.insert("f", "f", 0); // f is not overwritten + + tree.flush_active_memtable(0)?; + tree.major_compact(1, 0)?; + + tree.insert("a", "a", 1); + tree.insert("b", "b", 1); + tree.insert("c", "c", 1); + tree.insert("d", "d", 1); + tree.insert("e", "e", 1); + + tree.flush_active_memtable(0)?; + tree.major_compact(1, 10)?; + + assert_eq!(6, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + tree.major_compact(1, 11)?; + + assert_eq!(6, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + for table in tree.current_version().iter_tables() { + assert_eq!( + 1, + table + .list_blob_file_references()? + .unwrap() + .iter() + .map(|x| x.len) + .sum::(), + ); + assert_eq!( + 1, + table + .list_blob_file_references()? + .unwrap() + .iter() + .map(|x| x.bytes) + .sum::(), + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_sep_threshold.rs b/crates/lsm-tree/tests/blob_sep_threshold.rs new file mode 100644 index 000000000..81e50b109 --- /dev/null +++ b/crates/lsm-tree/tests/blob_sep_threshold.rs @@ -0,0 +1,30 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_separation_threshold() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().separation_threshold(1_024), + )) + .open()?; + + tree.insert("a", "a".repeat(1_023), 0); + tree.flush_active_memtable(0)?; + assert_eq!(0, tree.blob_file_count()); + + tree.insert("b", "b".repeat(1_024), 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.blob_file_count()); + + assert_eq!(2, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_simple.rs b/crates/lsm-tree/tests/blob_simple.rs new file mode 100644 index 000000000..852cb2873 --- /dev/null +++ b/crates/lsm-tree/tests/blob_simple.rs @@ -0,0 +1,77 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_simple_flush_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let new_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("smol", "small value", 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + assert_eq!( + big_value.len() as u32, + tree.size_of("big", SeqNo::MAX)?.unwrap(), + ); + + tree.flush_active_memtable(0)?; + + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"small value"); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.insert("big", &new_big_value, 1); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + + tree.flush_active_memtable(0)?; + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + + let value = tree.get("big", 1)?.expect("should exist"); + assert_eq!(&*value, big_value); + + let g = tree.prefix("big", 1, None).next().expect("should exist"); + assert_eq!(big_value.len() as u32, g.size()?); + } + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + let value = tree.get("smol", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"small value"); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, new_big_value); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_tree_guarded_size.rs b/crates/lsm-tree/tests/blob_tree_guarded_size.rs new file mode 100644 index 000000000..4a5d80244 --- /dev/null +++ b/crates/lsm-tree/tests/blob_tree_guarded_size.rs @@ -0,0 +1,25 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn blob_tree_guarded_size() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("a".as_bytes(), "abc", 0); + tree.insert("b".as_bytes(), "a".repeat(10_000), 0); + + assert_eq!( + 10_003u32, + tree.iter(SeqNo::MAX, None).flat_map(Guard::size).sum(), + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/blob_tree_reload_blob.rs b/crates/lsm-tree/tests/blob_tree_reload_blob.rs new file mode 100644 index 000000000..56605295f --- /dev/null +++ b/crates/lsm-tree/tests/blob_tree_reload_blob.rs @@ -0,0 +1,144 @@ +use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter, TreeType, +}; +use test_log::test; + +const ITEM_COUNT: usize = 10_000; + +#[test] +fn blob_tree_reload_empty() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .map(|x| x.key()) + .rev() + .flatten() + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Blob); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .map(|x| x.key()) + .rev() + .flatten() + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Blob); + + tree.flush_active_memtable(0)?; + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .map(|x| x.key()) + .rev() + .flatten() + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Blob); + } + + Ok(()) +} + +#[test] +fn blob_tree_reload() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + { + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .with_kv_separation(Some(Default::default())) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), seqno.next()); + } + + tree.flush_active_memtable(0)?; + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT * 2 + ); + + tree.flush_active_memtable(0)?; + } + + { + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT * 2 + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_filter.rs b/crates/lsm-tree/tests/compaction_filter.rs new file mode 100644 index 000000000..0b7fdf682 --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter.rs @@ -0,0 +1,199 @@ +use lsm_tree::compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, +}; +use lsm_tree::compaction::PullDown; +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter}; +use std::sync::{Arc, Mutex}; +use test_log::test; + +fn u32_s(n: u32) -> [u8; 4] { + n.to_be_bytes() +} + +fn u32_f(mut buf: &[u8]) -> u32 { + use byteorder::{ReadBytesExt, BE}; + + buf.read_u32::().unwrap() +} + +fn filter_basic(blob: bool) -> lsm_tree::Result<()> { + struct FilterState { + expect_blob: bool, + disable: bool, + do_rewrite: bool, + saw_value: bool, + drop_threshold: u32, + finished: bool, + expect_last_level: Option, + } + + let filter_state = Arc::new(Mutex::new(FilterState { + expect_blob: true, + disable: true, + do_rewrite: false, + saw_value: false, + drop_threshold: 4, + finished: false, + expect_last_level: None, + })); + + struct Filter(Arc>); + + impl CompactionFilter for Filter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + let mut state = self.0.lock().unwrap(); + if state.disable { + return Ok(Verdict::Keep); + } + + let key = u32_f(item.key()); + + if key == 0x00abcdef { + Ok(Verdict::RemoveWeak) + } else if key == 0x01abcdef { + Ok(Verdict::Destroy) + } else if key >= 0xff000000 { + let value = u32_f(&item.value()?); + assert_eq!(key & 0xff, value); + assert_eq!(item.is_indirection(), state.expect_blob); + state.saw_value = true; + + if !state.do_rewrite { + Ok(Verdict::Keep) + } else { + Ok(Verdict::ReplaceValue(vec![b'a'; value as usize].into())) + } + } else { + assert_eq!(item.value().expect("failed fetch"), b"a"); + if key < state.drop_threshold { + Ok(Verdict::Remove) + } else { + Ok(Verdict::Keep) + } + } + } + + fn finish(self: Box) { + let mut state = self.0.lock().unwrap(); + state.finished = true; + } + } + + struct FilterFactory(Arc>); + + impl Factory for FilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, ctx: &CompactionFilterContext) -> Box { + { + let guard = self.0.lock().unwrap(); + if let Some(expect_last_level) = guard.expect_last_level { + assert_eq!(ctx.is_last_level, expect_last_level) + } + } + Box::new(Filter(self.0.clone())) + } + } + + let folder = get_tmp_folder(); + + let mut config = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(if blob { + Some(KvSeparationOptions::default().separation_threshold(2)) + } else { + None + }) + .with_compaction_filter_factory(Some(Arc::new(FilterFactory(filter_state.clone())))); + config.level_count = 3; + let tree = config.open()?; + + for i in 0u32..255 { + tree.insert(u32_s(i), "old", 0); + } + + tree.flush_active_memtable(0)?; + tree.major_compact(u64::MAX, SeqNo::MAX)?; + + for i in 0u32..255 { + tree.insert(u32_s(i), "a", 0); + tree.insert(u32_s(i | 0xff000000), u32_s(i), 0); + } + + tree.insert(u32_s(0x00abcdef), "test", 0); + tree.insert(u32_s(0x01abcdef), "test", 0); + + tree.flush_active_memtable(0)?; + + let mut state = filter_state.lock().unwrap(); + state.disable = false; + state.expect_blob = blob; + state.expect_last_level = Some(false); + drop(state); + + tree.compact(Arc::new(PullDown(0, 1)), SeqNo::MAX)?; + + // filter should have dropped this + assert!(tree.get(u32_s(2), SeqNo::MAX)?.is_none()); + + // but not these + assert!(tree.get(u32_s(5), SeqNo::MAX)?.is_some()); + assert!(tree.get(u32_s(12), SeqNo::MAX)?.is_some()); + + // ensure weak-deleted value is gone + assert!(tree.get(u32_s(0x00abcdef), SeqNo::MAX)?.is_none()); + // ensure destroyed value is gone + assert!(tree.get(u32_s(0x01abcdef), SeqNo::MAX)?.is_none()); + + let mut state = filter_state.lock().unwrap(); + // filter should think it was called + assert!(state.saw_value); + assert!(state.finished); + state.saw_value = false; + // up the threshold + state.drop_threshold = 8; + state.do_rewrite = true; + state.expect_last_level = Some(true); + drop(state); + + // compact to last level + tree.major_compact(u64::MAX, SeqNo::MAX)?; + + // verify values gone + assert!(tree.get(u32_s(2), SeqNo::MAX)?.is_none()); + assert!(tree.get(u32_s(5), SeqNo::MAX)?.is_none()); + // ensure the other value isn't + assert!(tree.get(u32_s(12), SeqNo::MAX)?.is_some()); + + // verify rewrites work + assert_eq!(tree.get(u32_s(2 | 0xff000000), SeqNo::MAX)?.unwrap(), b"aa"); + assert_eq!( + tree.get(u32_s(37 | 0xff000000), SeqNo::MAX)?.unwrap(), + b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ); + + let state = filter_state.lock().unwrap(); + assert!(state.saw_value); + drop(state); + + Ok(()) +} + +#[test] +fn compaction_filter_with_blob() -> lsm_tree::Result<()> { + filter_basic(true) +} + +#[test] +fn compaction_filter_no_blob() -> lsm_tree::Result<()> { + filter_basic(false) +} diff --git a/crates/lsm-tree/tests/compaction_filter_gc_stats_change.rs b/crates/lsm-tree/tests/compaction_filter_gc_stats_change.rs new file mode 100644 index 000000000..d9a146c6c --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter_gc_stats_change.rs @@ -0,0 +1,263 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, + compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, + }, + get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn compaction_filter_gc_stats_change() -> lsm_tree::Result<()> { + struct MyFilter; + + impl CompactionFilter for MyFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key() == b"changethis" { + Ok(Verdict::ReplaceValue(b"winter2".repeat(128_000).into())) + } else { + Ok(Verdict::Keep) + } + } + } + + struct MyFilterFactory; + + impl Factory for MyFilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(MyFilter) + } + } + + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let other_big_value = b"asdasd".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .with_compaction_filter_factory(Some(Arc::new(MyFilterFactory))) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("changethis", &other_big_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + // Because we are creating a new value, a new blob file is created + assert_eq!(2, tree.blob_file_count()); + + let value = tree.get("changethis", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"winter2".repeat(128_000)); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "changethis":0 was changed + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = other_big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + assert_eq!(other_big_value.len() as u64, tree.stale_blob_bytes()); + } + + Ok(()) +} + +#[test] +fn compaction_filter_gc_stats_change_non_blob() -> lsm_tree::Result<()> { + struct MyFilter; + + impl CompactionFilter for MyFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key() == b"changethis" { + Ok(Verdict::ReplaceValue(b"winter2".into())) + } else { + Ok(Verdict::Keep) + } + } + } + + struct MyFilterFactory; + + impl Factory for MyFilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(MyFilter) + } + } + + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let small_value = b"asdasd"; + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .with_compaction_filter_factory(Some(Arc::new(MyFilterFactory))) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("changethis", small_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + // Because we are creating a new value, but a small one, a new blob file is not created + assert_eq!(1, tree.blob_file_count()); + + let value = tree.get("changethis", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"winter2"); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "changethis":0 was changed - but not a blob + assert_eq!(&lsm_tree::HashMap::default(), &*gc_stats,); + assert_eq!(0, tree.stale_blob_bytes()); + } + + Ok(()) +} + +#[test] +fn compaction_filter_gc_stats_change_blob_writer_rotation() -> lsm_tree::Result<()> { + struct MyFilter; + + impl CompactionFilter for MyFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key().starts_with(b"changethis") { + Ok(Verdict::ReplaceValue(b"winter2".repeat(128_000).into())) + } else { + Ok(Verdict::Keep) + } + } + } + + struct MyFilterFactory; + + impl Factory for MyFilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(MyFilter) + } + } + + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let other_big_value = b"asdasd".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .file_target_size(1) + .compression(lsm_tree::CompressionType::None), + )) + .with_compaction_filter_factory(Some(Arc::new(MyFilterFactory))) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("changethis", &other_big_value, 0); + tree.insert("changethis2", &other_big_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(3, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + // Because we are creating a new value, a new blob file is created + assert_eq!(5, tree.blob_file_count()); + + let value = tree.get("changethis", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"winter2".repeat(128_000)); + + let value = tree.get("changethis2", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, b"winter2".repeat(128_000)); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "changethis":0,"changethis2":0 were changed + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = other_big_value.len() as u64; + map.insert(1, FragmentationEntry::new(1, size, size)); + map.insert(2, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + assert_eq!(2 * other_big_value.len() as u64, tree.stale_blob_bytes()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_filter_gc_stats_destroy.rs b/crates/lsm-tree/tests/compaction_filter_gc_stats_destroy.rs new file mode 100644 index 000000000..f9f5b42b7 --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter_gc_stats_destroy.rs @@ -0,0 +1,90 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, + compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, + }, + get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use std::sync::Arc; +use test_log::test; + +struct MyFilter; + +impl CompactionFilter for MyFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key() == b"removethis" { + Ok(Verdict::Destroy) + } else { + Ok(Verdict::Keep) + } + } +} + +struct MyFilterFactory; + +impl Factory for MyFilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(MyFilter) + } +} + +#[test] +fn compaction_filter_gc_stats_destroy() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let other_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .with_compaction_filter_factory(Some(Arc::new(MyFilterFactory))) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("removethis", &other_big_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "removethis":0 is dropped/destroyed + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = other_big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + assert_eq!(other_big_value.len() as u64, tree.stale_blob_bytes()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_filter_gc_stats_remove.rs b/crates/lsm-tree/tests/compaction_filter_gc_stats_remove.rs new file mode 100644 index 000000000..3cd7b3962 --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter_gc_stats_remove.rs @@ -0,0 +1,90 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, + compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, + }, + get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use std::sync::Arc; +use test_log::test; + +struct MyFilter; + +impl CompactionFilter for MyFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key() == b"removethis" { + Ok(Verdict::Remove) + } else { + Ok(Verdict::Keep) + } + } +} + +struct MyFilterFactory; + +impl Factory for MyFilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(MyFilter) + } +} + +#[test] +fn compaction_filter_gc_stats_remove() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let other_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .with_compaction_filter_factory(Some(Arc::new(MyFilterFactory))) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("removethis", &other_big_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "removethis":0 is removed + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = other_big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + assert_eq!(other_big_value.len() as u64, tree.stale_blob_bytes()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_filter_gc_stats_remove_weak.rs b/crates/lsm-tree/tests/compaction_filter_gc_stats_remove_weak.rs new file mode 100644 index 000000000..0f7749fcf --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter_gc_stats_remove_weak.rs @@ -0,0 +1,90 @@ +use lsm_tree::{ + blob_tree::FragmentationEntry, + compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, + }, + get_tmp_folder, AbstractTree, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use std::sync::Arc; +use test_log::test; + +struct MyFilter; + +impl CompactionFilter for MyFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + if item.key() == b"removethis" { + Ok(Verdict::RemoveWeak) + } else { + Ok(Verdict::Keep) + } + } +} + +struct MyFilterFactory; + +impl Factory for MyFilterFactory { + fn name(&self) -> &str { + "Test" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(MyFilter) + } +} + +#[test] +fn compaction_filter_gc_stats_remove_weak() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let big_value = b"neptune!".repeat(128_000); + let other_big_value = b"winter!".repeat(128_000); + + { + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default().compression(lsm_tree::CompressionType::None), + )) + .with_compaction_filter_factory(Some(Arc::new(MyFilterFactory))) + .open()?; + + assert!(tree.get("big", SeqNo::MAX)?.is_none()); + tree.insert("big", &big_value, 0); + tree.insert("removethis", &other_big_value, 0); + + let value = tree.get("big", SeqNo::MAX)?.expect("should exist"); + assert_eq!(&*value, big_value); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.major_compact(64_000_000, 1_000)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + let gc_stats = tree.current_version().gc_stats().clone(); + + // "removethis":0 is removed + assert_eq!( + &{ + let mut map = lsm_tree::HashMap::default(); + let size = other_big_value.len() as u64; + map.insert(0, FragmentationEntry::new(1, size, size)); + map + }, + &*gc_stats, + ); + assert_eq!(other_big_value.len() as u64, tree.stale_blob_bytes()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_filter_snapshot.rs b/crates/lsm-tree/tests/compaction_filter_snapshot.rs new file mode 100644 index 000000000..d93d1f758 --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter_snapshot.rs @@ -0,0 +1,56 @@ +use lsm_tree::compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, +}; +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +struct NukeFilter; + +impl CompactionFilter for NukeFilter { + fn filter_item( + &mut self, + _: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + // data? what data? + Ok(Verdict::Remove) + } +} + +struct NukeFilterFactory; + +impl Factory for NukeFilterFactory { + fn name(&self) -> &str { + "Nuke" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(NukeFilter) + } +} + +#[test] +fn compaction_filter_snapshot() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let config = lsm_tree::Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .with_compaction_filter_factory(Some(Arc::new(NukeFilterFactory))); + let tree = config.open()?; + + tree.insert("a", "a", seqno.next()); + tree.flush_active_memtable(0)?; + tree.insert("b", "b", seqno.next()); + tree.flush_active_memtable(0)?; + + let snapshot_seqno = seqno.get(); + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + tree.major_compact(u64::MAX, 0)?; + + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + assert!(tree.get("a", SeqNo::MAX)?.is_none()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_filter_ttl.rs b/crates/lsm-tree/tests/compaction_filter_ttl.rs new file mode 100644 index 000000000..cc2ac755d --- /dev/null +++ b/crates/lsm-tree/tests/compaction_filter_ttl.rs @@ -0,0 +1,72 @@ +use lsm_tree::{ + compaction::filter::{ + CompactionFilter, Context as CompactionFilterContext, Factory, ItemAccessor, Verdict, + }, + AbstractTree, Config, +}; +use std::sync::{ + atomic::{AtomicU64, Ordering::Relaxed}, + Arc, +}; +use test_log::test; + +#[test] +fn compaction_filter_ttl() -> lsm_tree::Result<()> { + let folder = tempfile::tempdir()?; + + struct TtlFilter(u64); + + impl CompactionFilter for TtlFilter { + fn filter_item( + &mut self, + item: ItemAccessor<'_>, + _ctx: &CompactionFilterContext, + ) -> lsm_tree::Result { + let watermark_bytes = &self.0.to_be_bytes(); + + if item.key() < watermark_bytes { + Ok(Verdict::Destroy) + } else { + Ok(Verdict::Keep) + } + } + } + + #[derive(Default)] + struct TtlFilterFactory(Arc); + + impl Factory for TtlFilterFactory { + fn name(&self) -> &str { + "TTL" + } + + fn make_filter(&self, _ctx: &CompactionFilterContext) -> Box { + Box::new(TtlFilter(self.0.load(Relaxed))) + } + } + + let watermark: Arc = Arc::default(); + let factory = TtlFilterFactory(watermark.clone()); + + let tree = Config::new(folder, Default::default(), Default::default()) + .with_compaction_filter_factory(Some(Arc::new(factory))) + .open()?; + + tree.insert(1_000u64.to_be_bytes(), "my_value", 0); + tree.insert(1_001u64.to_be_bytes(), "my_value2", 1); + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.len(100_000, None)?); + + tree.major_compact(64_000_000, 100_000)?; + assert_eq!(2, tree.len(100_000, None)?); + + watermark.store(1_000, Relaxed); + tree.major_compact(64_000_000, 100_000)?; + assert_eq!(2, tree.len(100_000, None)?); + + watermark.store(1_001, Relaxed); + tree.major_compact(64_000_000, 100_000)?; + assert_eq!(1, tree.len(100_000, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/compaction_readers_grouping.rs b/crates/lsm-tree/tests/compaction_readers_grouping.rs new file mode 100644 index 000000000..2fc000bda --- /dev/null +++ b/crates/lsm-tree/tests/compaction_readers_grouping.rs @@ -0,0 +1,64 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +/// NOTE: Fix: https://github.com/fjall-rs/lsm-tree/commit/66a974ae6748646a40df475c291e04cf1dfbaece +#[test] +fn compaction_readers_grouping() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + let path = folder.path(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(path, seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a".as_bytes(), "abc", seqno.next()); + tree.insert("b".as_bytes(), "abc", seqno.next()); + tree.insert("c".as_bytes(), "abc", seqno.next()); + tree.flush_active_memtable(0)?; + assert_eq!(3, tree.len(SeqNo::MAX, None)?); + + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 2)), 0)?; + + tree.insert("d".as_bytes(), "abc", seqno.next()); + tree.insert("e".as_bytes(), "abc", seqno.next()); + tree.insert("f".as_bytes(), "abc", seqno.next()); + tree.flush_active_memtable(0)?; + assert_eq!(6, tree.len(SeqNo::MAX, None)?); + + tree.insert("g".as_bytes(), "abc", seqno.next()); + tree.insert("h".as_bytes(), "abc", seqno.next()); + tree.insert("i".as_bytes(), "abc", seqno.next()); + tree.flush_active_memtable(0)?; + assert_eq!(9, tree.len(SeqNo::MAX, None)?); + + // NOTE: Previously, create_compaction_stream would short circuit + // breaking this + tree.compact(Arc::new(lsm_tree::compaction::PullDown(2, 3)), 0)?; + + assert!(!tree + .current_version() + .level(0) + .expect("level should exist") + .is_empty()); + + assert!(tree + .current_version() + .level(1) + .expect("level should exist") + .is_empty()); + + assert!(tree + .current_version() + .level(2) + .expect("level should exist") + .is_empty()); + + assert!(!tree + .current_version() + .level(3) + .expect("level should exist") + .is_empty()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/dirty_snapshot.rs b/crates/lsm-tree/tests/dirty_snapshot.rs new file mode 100644 index 000000000..018318eab --- /dev/null +++ b/crates/lsm-tree/tests/dirty_snapshot.rs @@ -0,0 +1,24 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn dirty_snapshot_after_drop_range() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(dir.path(), seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a", "a", seqno.next()); + tree.flush_active_memtable(0)?; + tree.insert("b", "b", seqno.next()); + tree.flush_active_memtable(0)?; + + let snapshot_seqno = seqno.get(); + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + tree.drop_range("a"..="a")?; + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/expect_point_read_hits.rs b/crates/lsm-tree/tests/expect_point_read_hits.rs new file mode 100644 index 000000000..f5e63ee5f --- /dev/null +++ b/crates/lsm-tree/tests/expect_point_read_hits.rs @@ -0,0 +1,48 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_builds_filters() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .expect_point_read_hits(false) + .open()?; + + tree.insert("a", "a", 0); + + tree.flush_active_memtable(0)?; + assert!(tree.filter_size() > 0); + + tree.major_compact(u64::MAX, 0)?; + assert!(tree.filter_size() > 0); + + Ok(()) +} + +#[test] +fn tree_expect_point_read_hits() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .expect_point_read_hits(true) + .open()?; + + tree.insert("a", "a", 0); + + tree.flush_active_memtable(0)?; + assert!(tree.filter_size() > 0); + + tree.major_compact(u64::MAX, 0)?; + assert!(tree.filter_size() == 0); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/fifo_ttl.rs b/crates/lsm-tree/tests/fifo_ttl.rs new file mode 100644 index 000000000..7b3e4533e --- /dev/null +++ b/crates/lsm-tree/tests/fifo_ttl.rs @@ -0,0 +1,134 @@ +use lsm_tree::{ + compaction::Fifo, get_tmp_folder, AbstractTree, Config, KvSeparationOptions, + SequenceNumberCounter, +}; +use std::sync::Arc; + +#[test] +fn fifo_ttl_no_drop_when_recent_or_disabled() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Two quick tables (both recent) + tree.insert("a", "1", 0); + tree.flush_active_memtable(0)?; + tree.insert("b", "2", 1); + tree.flush_active_memtable(1)?; + + assert_eq!(2, tree.table_count()); + + // TTL enabled but not yet expired + let fifo_recent = Arc::new(Fifo::new(u64::MAX, Some(15))); + tree.compact(fifo_recent, 2)?; + assert_eq!(2, tree.table_count()); + + // TTL disabled explicitly + let fifo_disabled = Arc::new(Fifo::new(u64::MAX, None)); + tree.compact(fifo_disabled, 2)?; + assert_eq!(2, tree.table_count()); + + Ok(()) +} + +#[test] +fn fifo_below_limit_no_drop_standard() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..3u8 { + tree.insert([b'k', i].as_slice(), "v", i as u64); + tree.flush_active_memtable(i as u64)?; + } + + let fifo = Arc::new(Fifo::new(u64::MAX, None)); + tree.compact(fifo, 3)?; + + assert_eq!(3, tree.table_count()); + + Ok(()) +} + +#[test] +fn fifo_limit_considers_blob_bytes() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + // Create multiple small tables referencing blob files + for i in 0..3u8 { + tree.insert([b'k', i].as_slice(), "$", i as u64); // value goes to blob + tree.flush_active_memtable(i as u64)?; + } + + let before = tree.table_count(); + assert_eq!(3, before); + + // Very small limit forces dropping based on (table + blob) size + let fifo = Arc::new(Fifo::new(10, None)); + tree.compact(fifo, 3)?; + + // Should have dropped at least one table due to blob bytes pressure + assert!(tree.table_count() < before); + + Ok(()) +} + +#[test] +fn fifo_compact_empty_tree_noop() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let fifo = Arc::new(Fifo::new(1_000_000, Some(1))); + tree.compact(fifo, 0)?; + + assert_eq!(0, tree.table_count()); + + Ok(()) +} + +#[test] +fn fifo_idempotent_when_within_limits() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + let tree = Config::new( + dir.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..3u8 { + tree.insert([b'k', i].as_slice(), "v", i as u64); + tree.flush_active_memtable(i as u64)?; + } + + let fifo = Arc::new(Fifo::new(u64::MAX, None)); + tree.compact(fifo.clone(), 3)?; + let after_first = tree.table_count(); + + tree.compact(fifo, 3)?; + let after_second = tree.table_count(); + + assert_eq!(after_first, after_second); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/guard_if.rs b/crates/lsm-tree/tests/guard_if.rs new file mode 100644 index 000000000..6072d5d0f --- /dev/null +++ b/crates/lsm-tree/tests/guard_if.rs @@ -0,0 +1,71 @@ +use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, Guard, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn guard_into_inner_if() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("earth#name", "earth", 0); + tree.insert("earth#color", "BLUE", 0); + + assert_eq!(2, tree.iter(SeqNo::MAX, None).count()); + + assert_eq!( + 1, + tree.iter(SeqNo::MAX, None) + .filter_map(|guard| { + guard + .into_inner_if(|key| key.ends_with(b"#name")) + .unwrap() + .1 + }) + .count(), + ); + } + + Ok(()) +} + +#[test] +fn guard_into_inner_if_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + tree.insert("earth#name", "earth", 0); + tree.insert("earth#color", "BLUE", 0); + + assert_eq!(2, tree.iter(SeqNo::MAX, None).count()); + + assert_eq!( + 1, + tree.iter(SeqNo::MAX, None) + .filter_map(|guard| { + guard + .into_inner_if(|key| key.ends_with(b"#name")) + .unwrap() + .1 + }) + .count(), + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/ingest_dirty_snapshot.rs b/crates/lsm-tree/tests/ingest_dirty_snapshot.rs new file mode 100644 index 000000000..6bf1a6d7d --- /dev/null +++ b/crates/lsm-tree/tests/ingest_dirty_snapshot.rs @@ -0,0 +1,23 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; + +#[test] +fn ingestion_dirty_snapshot() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a", "a", seqno.next()); + tree.insert("a", "b", seqno.next()); + + let snapshot_seqno = 1; + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + let mut ingest = tree.ingestion()?; + ingest.write("b", "b")?; + ingest.finish()?; + + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/ingestion_api.rs b/crates/lsm-tree/tests/ingestion_api.rs new file mode 100644 index 000000000..27ba70d1b --- /dev/null +++ b/crates/lsm-tree/tests/ingestion_api.rs @@ -0,0 +1,368 @@ +use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; + +#[test] +fn tree_ingestion_tombstones_delete_existing_keys() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..10u32 { + let key = format!("k{:03}", i); + tree.insert(key.as_bytes(), b"v", 0); + } + + let mut ingest = tree.ingestion()?; + for i in 0..10u32 { + let key = format!("k{:03}", i); + ingest.write_tombstone(key)?; + } + ingest.finish()?; + + for i in 0..10u32 { + let key = format!("k{:03}", i); + assert!(tree.get(key.as_bytes(), SeqNo::MAX)?.is_none()); + } + assert_eq!(tree.tombstone_count(), 10); + + Ok(()) +} + +#[test] +fn sealed_memtable_value_overrides_table_value() -> lsm_tree::Result<()> { + use lsm_tree::AbstractTree; + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Older table value via ingestion (seqno 1) + { + let mut ingest = tree.ingestion()?; + ingest.write(b"k", b"old")?; + ingest.finish()?; + } + + // Newer value in memtable (seqno 2), then seal it + tree.insert(b"k", b"new", 2); + let _ = tree.rotate_memtable(); // move active -> sealed + + // Read should return the sealed memtable value + assert_eq!( + tree.get(b"k", lsm_tree::SeqNo::MAX)?, + Some(b"new".as_slice().into()) + ); + + Ok(()) +} + +#[test] +fn sealed_memtable_tombstone_overrides_table_value() -> lsm_tree::Result<()> { + use lsm_tree::AbstractTree; + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Older table value via ingestion (seqno 1) + { + let mut ingest = tree.ingestion()?; + ingest.write(b"k", b"old")?; + ingest.finish()?; + } + + // Newer tombstone in memtable (seqno 2), then seal it + tree.remove(b"k", 2); + let _ = tree.rotate_memtable(); + + // Read should see the delete from sealed memtable + assert!(tree.get(b"k", lsm_tree::SeqNo::MAX)?.is_none()); + + Ok(()) +} + +#[test] +fn tables_newest_first_returns_highest_seqno() -> lsm_tree::Result<()> { + use lsm_tree::AbstractTree; + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Two separate ingestions create two tables containing the same key at different seqnos + { + let mut ingest = tree.ingestion()?; + ingest.write(b"k", b"v1")?; + ingest.finish()?; + } + { + let mut ingest = tree.ingestion()?; + ingest.write(b"k", b"v2")?; + ingest.finish()?; + } + + // With memtables empty, read should return the value from the newest table run (seqno 2) + assert_eq!( + tree.get(b"k", lsm_tree::SeqNo::MAX)?, + Some(b"v2".as_slice().into()) + ); + Ok(()) +} + +#[test] +#[should_panic(expected = "next key in ingestion must be greater than last key")] +fn ingestion_enforces_order_standard_panics() { + let folder = tempfile::tempdir().unwrap(); + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open() + .unwrap(); + + let mut ingest = tree.ingestion().unwrap(); + + // First write higher key, then lower to trigger ordering assertion + ingest.write(b"k2", b"v").unwrap(); + + // Panics here + let _ = ingest.write(b"k1", b"v"); +} + +#[test] +fn blob_ingestion_out_of_order_panics_without_blob_write() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(8))) + .open()?; + + let before = tree.blob_file_count(); + + // Use a small value for the first write to avoid blob I/O + let result = std::panic::catch_unwind(|| { + let mut ingest = tree.ingestion().unwrap(); + ingest.write(b"k2", b"x").unwrap(); + + // Second write would require blob I/O, but ordering check should fire before any blob write + let _ = ingest.write(b"k1", [1u8; 16]); + }); + assert!(result.is_err()); + + let after = tree.blob_file_count(); + assert_eq!(before, after); + + Ok(()) +} + +#[test] +fn memtable_put_overrides_table_tombstone() -> lsm_tree::Result<()> { + use lsm_tree::AbstractTree; + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Older put written via ingestion to tables (seqno 1) + { + let mut ingest = tree.ingestion()?; + ingest.write(b"k", b"v1")?; + ingest.finish()?; + } + + // Newer tombstone written via ingestion to tables (seqno 2) + { + let mut ingest = tree.ingestion()?; + ingest.write_tombstone(b"k")?; + ingest.finish()?; + } + + // Newest put in active memtable (seqno 3) should override table tombstone + tree.insert(b"k", b"v3", 3); + assert_eq!( + tree.get(b"k", lsm_tree::SeqNo::MAX)?, + Some(b"v3".as_slice().into()) + ); + Ok(()) +} + +#[test] +fn blob_tree_ingestion_tombstones_delete_existing_keys() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + for i in 0..8u32 { + let key = format!("b{:03}", i); + tree.insert(key.as_bytes(), b"x", 0); + } + + let mut ingest = tree.ingestion()?; + for i in 0..8u32 { + let key = format!("b{:03}", i); + ingest.write_tombstone(key)?; + } + ingest.finish()?; + + for i in 0..8u32 { + let key = format!("b{:03}", i); + assert!(tree.get(key.as_bytes(), SeqNo::MAX)?.is_none()); + } + assert_eq!(tree.tombstone_count(), 8); + + Ok(()) +} + +#[test] +fn tree_ingestion_finish_no_writes_noop() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let before_tables = tree.table_count(); + tree.ingestion()?.finish()?; + let after_tables = tree.table_count(); + + assert_eq!(before_tables, after_tables); + assert!(tree.is_empty(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn blob_ingestion_only_tombstones_does_not_create_blob_files() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + for i in 0..5u32 { + let key = format!("d{:03}", i); + tree.insert(key.as_bytes(), b"value", 0); + } + + let before_blobs = tree.blob_file_count(); + + let mut ingest = tree.ingestion()?; + for i in 0..5u32 { + let key = format!("d{:03}", i); + ingest.write_tombstone(key)?; + } + ingest.finish()?; + + let after_blobs = tree.blob_file_count(); + assert_eq!(before_blobs, after_blobs); + + for i in 0..5u32 { + let key = format!("d{:03}", i); + assert!(tree.get(key.as_bytes(), SeqNo::MAX)?.is_none()); + } + + Ok(()) +} + +#[test] +fn blob_ingestion_finish_no_writes_noop() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let before_tables = tree.table_count(); + let before_blobs = tree.blob_file_count(); + + tree.ingestion()?.finish()?; + + let after_tables = tree.table_count(); + let after_blobs = tree.blob_file_count(); + + assert_eq!(before_tables, after_tables); + assert_eq!(before_blobs, after_blobs); + assert!(tree.is_empty(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn blob_ingestion_separates_large_values_and_reads_ok() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(8))) + .open()?; + + let mut ingest = tree.ingestion()?; + ingest.write("k_big1", [1u8; 16])?; + ingest.write("k_big2", [2u8; 32])?; + ingest.write("k_small", "abc")?; + ingest.finish()?; + + assert!(tree.blob_file_count() >= 1); + + assert_eq!( + tree.get("k_small", SeqNo::MAX)?, + Some(b"abc".as_slice().into()) + ); + assert_eq!( + tree.get("k_big1", SeqNo::MAX)?.as_deref().map(|s| s.len()), + Some(16) + ); + assert_eq!( + tree.get("k_big2", SeqNo::MAX)?.as_deref().map(|s| s.len()), + Some(32) + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/ingestion_invariants.rs b/crates/lsm-tree/tests/ingestion_invariants.rs new file mode 100644 index 000000000..1c9a94127 --- /dev/null +++ b/crates/lsm-tree/tests/ingestion_invariants.rs @@ -0,0 +1,181 @@ +use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +#[test] +fn ingestion_autoflushes_active_memtable() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Write to active memtable + for i in 0..10u32 { + let k = format!("a{:03}", i); + tree.insert(k.as_bytes(), b"v", 1); + } + + let tables_before = tree.table_count(); + let sealed_before = tree.sealed_memtable_count(); + assert_eq!(sealed_before, 0); + + // Start ingestion (should auto-flush active) + let mut ingest = tree.ingestion()?; + ingest.write("a", "a")?; + ingest.finish()?; + + // After ingestion, data is in tables; no sealed memtables + assert_eq!(tree.sealed_memtable_count(), 0); + assert!(tree.table_count() > tables_before); + + // Reads must succeed from tables + for i in 0..10u32 { + let k = format!("a{:03}", i); + assert_eq!( + tree.get(k.as_bytes(), SeqNo::MAX)?, + Some(b"v".as_slice().into()) + ); + } + + Ok(()) +} + +#[test] +fn ingestion_flushes_sealed_memtables() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Put items into active and seal them + for i in 0..8u32 { + let k = format!("s{:03}", i); + tree.insert(k.as_bytes(), b"x", 1); + } + assert!(tree.rotate_memtable().is_some()); + assert!(tree.sealed_memtable_count() > 0); + + let tables_before = tree.table_count(); + + // Ingestion should flush sealed memtables and register resulting tables + let mut ingest = tree.ingestion()?; + ingest.write("a", "a")?; + ingest.finish()?; + + assert_eq!(tree.sealed_memtable_count(), 0); + assert!(tree.table_count() > tables_before); + + for i in 0..8u32 { + let k = format!("s{:03}", i); + assert_eq!( + tree.get(k.as_bytes(), SeqNo::MAX)?, + Some(b"x".as_slice().into()) + ); + } + + Ok(()) +} + +#[test] +fn ingestion_blocks_memtable_writes_until_finish() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Acquire ingestion and keep it active while another thread performs writes + let ingest = tree.ingestion()?; + + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let tree2 = tree.clone(); + + let handle = thread::spawn(move || { + started_tx.send(()).ok(); + tree2.insert(b"k_block", b"v", 6); + done_tx.send(()).ok(); + }); + + // Wait for the writer thread to start the attempt + started_rx.recv().unwrap(); + + // Give it a moment; the insert should complete and not be blocked by ingestion + thread::sleep(Duration::from_millis(100)); + assert!(done_rx.try_recv().is_ok(), "insert should not be blocked"); + + handle.join().unwrap(); + ingest.finish()?; + + // Verify the write landed and is visible + assert_eq!( + tree.get(b"k_block", SeqNo::MAX)?, + Some(b"v".as_slice().into()) + ); + + Ok(()) +} + +#[test] +fn blob_ingestion_honors_invariants_and_blocks_writes() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + // Write small values into memtable and then start blob ingestion + for i in 0..4u32 { + let k = format!("b{:03}", i); + tree.insert(k.as_bytes(), b"y", 1); + } + + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let tree2 = tree.clone(); + + let ingest = tree.ingestion()?; + + let handle = thread::spawn(move || { + started_tx.send(()).ok(); + tree2.insert(b"b999", b"z", 31); + done_tx.send(()).ok(); + }); + + started_rx.recv().unwrap(); + thread::sleep(Duration::from_millis(100)); + assert!(done_rx.try_recv().is_ok()); + + handle.join().unwrap(); + ingest.finish()?; + + // Data visible after ingestion, including concurrent write + for i in 0..4u32 { + let k = format!("b{:03}", i); + assert_eq!( + tree.get(k.as_bytes(), SeqNo::MAX)?, + Some(b"y".as_slice().into()) + ); + } + assert_eq!(tree.get(b"b999", SeqNo::MAX)?, Some(b"z".as_slice().into())); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/ingestion_seqno.rs b/crates/lsm-tree/tests/ingestion_seqno.rs new file mode 100644 index 000000000..105965ed9 --- /dev/null +++ b/crates/lsm-tree/tests/ingestion_seqno.rs @@ -0,0 +1,25 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; + +#[test] +fn ingestion_persisted_seqno() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let mut ingest = tree.ingestion()?; + ingest.write("a", "a")?; + ingest.finish()?; + assert_eq!(Some(0), tree.get_highest_persisted_seqno()); + + let mut ingest = tree.ingestion()?; + ingest.write("b", "b")?; + ingest.finish()?; + assert_eq!(Some(1), tree.get_highest_persisted_seqno()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/latest_snapshot.rs b/crates/lsm-tree/tests/latest_snapshot.rs new file mode 100644 index 000000000..2ba104784 --- /dev/null +++ b/crates/lsm-tree/tests/latest_snapshot.rs @@ -0,0 +1,24 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn latest_snapshot() -> lsm_tree::Result<()> { + let dir = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(dir.path(), seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a", "a", seqno.next()); + tree.flush_active_memtable(0)?; + tree.insert("b", "b", seqno.next()); + tree.flush_active_memtable(0)?; + + tree.insert("c", "c", seqno.next()); + tree.insert("d", "d", seqno.next()); + + let snapshot_seqno = seqno.get(); + assert_eq!(None, tree.get("e", snapshot_seqno)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/leveled_trivial_move.rs b/crates/lsm-tree/tests/leveled_trivial_move.rs new file mode 100644 index 000000000..008c33ed5 --- /dev/null +++ b/crates/lsm-tree/tests/leveled_trivial_move.rs @@ -0,0 +1,60 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn leveled_trivial_move_into_l1() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let compaction = Arc::new(lsm_tree::compaction::Leveled::default()); + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(1, tree.level_table_count(6).unwrap_or_default()); + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(2, tree.level_table_count(6).unwrap_or_default()); + + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(3, tree.level_table_count(6).unwrap_or_default()); + + tree.insert("d", "d", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(0, tree.level_table_count(0).unwrap_or_default()); + assert_eq!(4, tree.level_table_count(6).unwrap_or_default()); + + // To keep runs minimal, we trivial move into L1 + tree.insert("e", "e", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(0, tree.level_table_count(0).unwrap_or_default()); + assert_eq!(5, tree.level_table_count(6).unwrap_or_default()); + + // To keep runs minimal, we trivial move into L1 + tree.insert("f", "f", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(0, tree.level_table_count(0).unwrap_or_default()); + assert_eq!(6, tree.level_table_count(6).unwrap_or_default()); + // Should not trivial move because overlap + tree.insert("d", "d", 0); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(1, tree.level_table_count(0).unwrap_or_default()); + assert_eq!(6, tree.level_table_count(6).unwrap_or_default()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/memtable_id.rs b/crates/lsm-tree/tests/memtable_id.rs new file mode 100644 index 000000000..a6aa670ea --- /dev/null +++ b/crates/lsm-tree/tests/memtable_id.rs @@ -0,0 +1,40 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +// https://github.com/fjall-rs/lsm-tree/issues/230 +#[test] +fn memtable_id() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + + assert_eq!( + 0, + tree.get_version_history_lock() + .latest_version() + .active_memtable + .id() + ); + + tree.flush_active_memtable(0)?; + + assert_eq!(1, tree.table_count()); + assert_eq!( + 1, + tree.get_version_history_lock() + .latest_version() + .active_memtable + .id() + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/model_1.rs b/crates/lsm-tree/tests/model_1.rs new file mode 100644 index 000000000..89108251e --- /dev/null +++ b/crates/lsm-tree/tests/model_1.rs @@ -0,0 +1,43 @@ +// Found by model testing + +use lsm_tree::{get_tmp_folder, AbstractTree, Result, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn model_1() -> Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + let compaction = Arc::new(lsm_tree::compaction::Leveled::default()); + + let value = b"hellohello"; + + tree.insert(b"a", value, 0); + tree.flush_active_memtable(0)?; + + tree.insert(b"b", value, 1); + tree.flush_active_memtable(0)?; + + tree.remove(b"b", 2); + tree.flush_active_memtable(0)?; + + tree.insert(b"c", value, 3); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + + { + log::info!(r#"Getting "b""#); + let seqno = 5; + assert!(!tree.contains_key(b"b", seqno)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/model_2.rs b/crates/lsm-tree/tests/model_2.rs new file mode 100644 index 000000000..1c84296c9 --- /dev/null +++ b/crates/lsm-tree/tests/model_2.rs @@ -0,0 +1,48 @@ +// Found by model testing + +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, Result, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn model_2() -> Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(5))) + .open()?; + let compaction = Arc::new(lsm_tree::compaction::Leveled::default()); + + let value = b"hellohello"; + + tree.insert("a", value, 3); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.blob_file_count()); + + tree.insert("b", value, 4); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(2, tree.blob_file_count()); + + tree.insert("a", value, 5); + tree.flush_active_memtable(0)?; + tree.compact(compaction.clone(), 0)?; + assert_eq!(3, tree.table_count()); + assert_eq!(3, tree.blob_file_count()); + + tree.drop_range::<&[u8], _>(..)?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/model_3.rs b/crates/lsm-tree/tests/model_3.rs new file mode 100644 index 000000000..df6d8c55b --- /dev/null +++ b/crates/lsm-tree/tests/model_3.rs @@ -0,0 +1,70 @@ +// Found by model testing + +use lsm_tree::{get_tmp_folder, AbstractTree, KvSeparationOptions, Result, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn model_3() -> Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(5))) + .open()?; + let compaction = Arc::new(lsm_tree::compaction::Leveled::default()); + + let value = b"hellohello"; + + tree.insert("a", value, 1); + tree.insert("i", value, 1); + tree.flush_active_memtable(0)?; + + tree.insert("a", value, 2); + tree.insert("f", value, 2); + tree.flush_active_memtable(0)?; + + tree.insert("a", value, 3); + tree.insert("h", value, 3); + tree.flush_active_memtable(0)?; + + tree.insert("a", value, 4); + tree.insert("b", value, 4); + tree.flush_active_memtable(0)?; + + tree.insert("c", value, 5); + tree.insert("g", value, 5); + tree.flush_active_memtable(0)?; + + tree.insert("b", value, 6); + tree.insert("c", value, 6); + tree.insert("d", value, 6); + tree.insert("e", value, 6); + tree.flush_active_memtable(15)?; + tree.compact(compaction.clone(), 41)?; + + tree.insert("a", value, 7); + tree.flush_active_memtable(16)?; + + tree.insert("a", value, 8); + tree.flush_active_memtable(17)?; + + tree.insert("a", value, 9); + tree.flush_active_memtable(18)?; + + tree.insert("a", value, 10); + tree.flush_active_memtable(19)?; + tree.compact(compaction.clone(), 19)?; + + tree.drop_range::<&[u8], _>(..)?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/model_4.rs b/crates/lsm-tree/tests/model_4.rs new file mode 100644 index 000000000..4789f3e93 --- /dev/null +++ b/crates/lsm-tree/tests/model_4.rs @@ -0,0 +1,112 @@ +// Found by model testing + +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, KvSeparationOptions, Result, + SequenceNumberCounter, +}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn model_4() -> Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(5))) + .data_block_size_policy(BlockSizePolicy::all(100)) + .open()?; + + let compaction = + Arc::new(lsm_tree::compaction::Leveled::default().with_table_target_size(1_000)); + + let value = b"hellohello"; + + tree.insert([0], value, 132); + tree.insert([0], value, 133); + tree.insert([1], value, 134); + tree.insert([2], value, 135); + tree.insert([3], value, 136); + tree.insert([4], value, 137); + tree.insert([5], value, 138); + tree.insert([6], value, 139); + tree.insert([7], value, 140); + tree.insert([0, 0, 0, 0, 0, 0, 9, 217], value, 141); + tree.insert([0, 0, 0, 0, 0, 0, 2, 77], value, 142); + tree.insert([0, 0, 0, 0, 0, 0, 33, 92], value, 143); + tree.insert([0, 0, 0, 0, 0, 0, 38, 41], value, 144); + tree.insert([0, 0, 0, 0, 0, 0, 22, 143], value, 145); + tree.insert([0, 0, 0, 0, 0, 0, 22, 161], value, 146); + tree.insert([0, 0, 0, 0, 0, 0, 9, 143], value, 148); + tree.insert([0, 0, 0, 0, 0, 0, 25, 222], value, 149); + tree.insert([0, 0, 0, 0, 0, 0, 11, 144], value, 150); + tree.insert([0, 0, 0, 0, 0, 0, 8, 208], value, 151); + tree.insert([0, 0, 0, 0, 0, 0, 31, 195], value, 152); + tree.insert([0, 0, 0, 0, 0, 0, 27, 47], value, 153); + tree.insert([0, 0, 0, 0, 0, 0, 31, 104], value, 154); + tree.insert([0, 0, 0, 0, 0, 0, 14, 219], value, 155); + tree.insert([0, 0, 0, 0, 0, 0, 17, 125], value, 156); + tree.insert([0, 0, 0, 0, 0, 0, 15, 52], value, 157); + tree.insert([0, 0, 0, 0, 0, 0, 20, 230], value, 158); + tree.insert([0, 0, 0, 0, 0, 0, 16, 88], value, 159); + tree.insert([0, 0, 0, 0, 0, 0, 9, 26], value, 160); + tree.insert([0, 0, 0, 0, 0, 0, 20, 21], value, 161); + tree.insert([0, 0, 0, 0, 0, 0, 27, 86], value, 162); + tree.insert([0, 0, 0, 0, 0, 0, 4, 112], value, 163); + tree.insert([0, 0, 0, 0, 0, 0, 12, 60], value, 164); + tree.insert([0, 0, 0, 0, 0, 0, 8, 186], value, 165); + tree.insert([0, 0, 0, 0, 0, 0, 34, 18], value, 166); + tree.insert([0, 0, 0, 0, 0, 0, 15, 156], value, 167); + tree.insert([0, 0, 0, 0, 0, 0, 5, 91], value, 168); + tree.insert([0, 0, 0, 0, 0, 0, 36, 0], value, 169); + tree.insert([0, 0, 0, 0, 0, 0, 38, 249], value, 170); + tree.insert([0, 0, 0, 0, 0, 0, 23, 42], value, 171); + tree.insert([0, 0, 0, 0, 0, 0, 23, 14], value, 172); + tree.insert([0, 0, 0, 0, 0, 0, 32, 119], value, 173); + tree.insert([0, 0, 0, 0, 0, 0, 31, 9], value, 174); + tree.insert([0, 0, 0, 0, 0, 0, 4, 170], value, 175); + tree.insert([0, 0, 0, 0, 0, 0, 18, 119], value, 176); + tree.insert([0, 0, 0, 0, 0, 0, 4, 178], value, 177); + tree.insert([0, 0, 0, 0, 0, 0, 4, 36], value, 178); + tree.insert([0, 0, 0, 0, 0, 0, 36, 53], value, 179); + tree.insert([0, 0, 0, 0, 0, 0, 35, 157], value, 181); + tree.insert([0, 0, 0, 0, 0, 0, 22, 24], value, 182); + tree.insert([0, 0, 0, 0, 0, 0, 33, 247], value, 183); + tree.insert([0, 0, 0, 0, 0, 0, 26, 236], value, 185); + tree.flush_active_memtable(86)?; + + tree.insert([0], value, 186); + tree.flush_active_memtable(87)?; + + tree.insert([0, 0, 0, 0, 0, 0, 7, 49], value, 187); + tree.flush_active_memtable(88)?; + + tree.insert([0, 0, 0, 0, 0, 0, 18, 134], value, 188); + tree.flush_active_memtable(89)?; + tree.compact(compaction.clone(), 89)?; + + tree.insert([0], value, 189); + tree.flush_active_memtable(90)?; + + tree.insert([0], value, 190); + tree.flush_active_memtable(91)?; + + tree.insert([0], value, 191); + tree.flush_active_memtable(92)?; + + tree.insert([0], value, 192); + tree.flush_active_memtable(93)?; + tree.compact(compaction.clone(), 93)?; + + tree.drop_range::<&[u8], _>(..)?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/model_5.rs b/crates/lsm-tree/tests/model_5.rs new file mode 100644 index 000000000..d683c387d --- /dev/null +++ b/crates/lsm-tree/tests/model_5.rs @@ -0,0 +1,87 @@ +// Found by model testing + +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, KvSeparationOptions, Result, + SequenceNumberCounter, +}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn model_5() -> Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(5))) + .data_block_size_policy(BlockSizePolicy::all(100)) + .open()?; + + let compaction = Arc::new(lsm_tree::compaction::Leveled::default().with_table_target_size(150)); + + let value = b"hellohello"; + + tree.insert([0], value, 753); + tree.insert([0, 0, 0, 0, 0, 0, 1, 0], value, 754); + tree.insert([0, 0, 0, 0, 0, 0, 0, 140], value, 756); + tree.insert([0, 0, 0, 0, 0, 0, 0, 127], value, 762); + tree.insert([0, 0, 0, 0, 0, 0, 0, 98], value, 763); + tree.insert([0, 0, 0, 0, 0, 0, 2, 138], value, 764); + tree.insert([0, 0, 0, 0, 0, 0, 3, 150], value, 765); + tree.insert([0, 0, 0, 0, 0, 0, 0, 23], value, 766); + tree.insert([0, 0, 0, 0, 0, 0, 3, 121], value, 767); + tree.insert([0, 0, 0, 0, 0, 0, 1, 212], value, 842); + tree.insert([0, 0, 0, 0, 0, 0, 2, 152], value, 843); + tree.insert([0, 0, 0, 0, 0, 0, 2, 241], value, 844); + tree.flush_active_memtable(798)?; + + tree.insert([0, 0, 0, 0, 0, 0, 3, 120], value, 898); + tree.flush_active_memtable(799)?; + + tree.insert([0, 0, 0, 0, 0, 0, 3, 89], value, 899); + tree.flush_active_memtable(800)?; + + tree.insert([0, 0, 0, 0, 0, 0, 1, 52], value, 901); + tree.insert([0, 0, 0, 0, 0, 0, 0, 177], value, 902); + tree.insert([0, 0, 0, 0, 0, 0, 3, 43], value, 903); + tree.insert([0, 0, 0, 0, 0, 0, 3, 41], value, 904); + tree.insert([0, 0, 0, 0, 0, 0, 3, 160], value, 905); + tree.insert([0, 0, 0, 0, 0, 0, 1, 182], value, 906); + tree.insert([0, 0, 0, 0, 0, 0, 0, 73], value, 907); + tree.insert([0, 0, 0, 0, 0, 0, 0, 78], value, 912); + tree.insert([0, 0, 0, 0, 0, 0, 2, 103], value, 913); + tree.insert([0, 0, 0, 0, 0, 0, 1, 39], value, 914); + tree.insert([0, 0, 0, 0, 0, 0, 1, 78], value, 927); + tree.insert([0, 0, 0, 0, 0, 0, 0, 244], value, 928); + tree.insert([0, 0, 0, 0, 0, 0, 2, 76], value, 929); + tree.insert([0, 0, 0, 0, 0, 0, 1, 202], value, 934); + tree.insert([0, 0, 0, 0, 0, 0, 2, 140], value, 936); + tree.insert([0, 0, 0, 0, 0, 0, 2, 152], value, 937); + tree.flush_active_memtable(886)?; + tree.compact(compaction.clone(), 886)?; + + tree.insert([0, 0, 0, 0, 0, 0, 3, 145], value, 989); + tree.flush_active_memtable(890)?; + + tree.insert([0, 0, 0, 0, 0, 0, 3, 99], value, 993); + tree.flush_active_memtable(894)?; + + tree.insert([0, 0, 0, 0, 0, 0, 1, 106], value, 997); + tree.flush_active_memtable(898)?; + + tree.insert([0, 0, 0, 0, 0, 0, 2, 99], value, 1001); + tree.flush_active_memtable(902)?; + tree.compact(compaction.clone(), 902)?; + + tree.drop_range::<&[u8], _>(..)?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/model_6.rs b/crates/lsm-tree/tests/model_6.rs new file mode 100644 index 000000000..b75f4f7e7 --- /dev/null +++ b/crates/lsm-tree/tests/model_6.rs @@ -0,0 +1,1293 @@ +// Found by model testing + +#![cfg_attr(rustfmt, rustfmt_skip)] + +use lsm_tree::{ + get_tmp_folder, + AbstractTree, KvSeparationOptions, Result, SequenceNumberCounter, config::{BlockSizePolicy, CompressionPolicy} +}; +use std::sync::Arc; +use test_log::test; + +// Yes this file is very large, it's hard to condense it to a more minimal repro +#[test] +fn model_6() -> Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new(path, SequenceNumberCounter::default(), SequenceNumberCounter::default()) + .data_block_compression_policy(CompressionPolicy::disabled()) + .index_block_compression_policy(CompressionPolicy::disabled()) + .data_block_size_policy(BlockSizePolicy::all(100)) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(10))) + .open()?; + + let compaction = Arc::new( + lsm_tree::compaction::Leveled::default() + .with_table_target_size(1_024) + ); + + tree.insert([0, 0, 0, 0, 0, 0, 3, 152], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170598); +tree.insert([0, 0, 0, 0, 0, 0, 0, 9], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170599); +tree.insert([0, 0, 0, 0, 0, 0, 1, 184], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170600); +tree.flush_active_memtable(170501)?; +tree.insert([0, 0, 0, 0, 0, 0, 3, 1], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170601); +tree.insert([0, 0, 0, 0, 0, 0, 2, 31], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170602); +tree.insert([0, 0, 0, 0, 0, 0, 3, 95], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170603); +tree.insert([0, 0, 0, 0, 0, 0, 1, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170604); +tree.insert([0, 0, 0, 0, 0, 0, 0, 40], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170605); +tree.insert([0, 0, 0, 0, 0, 0, 3, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170606); +tree.insert([0, 0, 0, 0, 0, 0, 2, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170607); +tree.insert([0, 0, 0, 0, 0, 0, 0, 4], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170608); +tree.insert([0, 0, 0, 0, 0, 0, 2, 151], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170609); +tree.insert([0, 0, 0, 0, 0, 0, 2, 4], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170610); +tree.insert([0, 0, 0, 0, 0, 0, 2, 99], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170611); +tree.insert([0, 0, 0, 0, 0, 0, 2, 133], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170612); +tree.insert([0, 0, 0, 0, 0, 0, 3, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170613); +tree.insert([0, 0, 0, 0, 0, 0, 2, 189], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170614); +tree.insert([0, 0, 0, 0, 0, 0, 3, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170615); +tree.insert([0, 0, 0, 0, 0, 0, 1, 157], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170616); +tree.insert([0, 0, 0, 0, 0, 0, 2, 102], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170617); +tree.insert([0, 0, 0, 0, 0, 0, 3, 119], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170619); +tree.insert([0, 0, 0, 0, 0, 0, 2, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170620); +tree.insert([0, 0, 0, 0, 0, 0, 0, 188], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170621); +tree.insert([0, 0, 0, 0, 0, 0, 0, 194], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170624); +tree.insert([0, 0, 0, 0, 0, 0, 1, 140], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170625); +tree.insert([0, 0, 0, 0, 0, 0, 0, 255], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170626); +tree.insert([0, 0, 0, 0, 0, 0, 1, 97], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170628); +tree.insert([0, 0, 0, 0, 0, 0, 2, 14], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170629); +tree.insert([0, 0, 0, 0, 0, 0, 0, 103], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170630); +tree.insert([0, 0, 0, 0, 0, 0, 0, 225], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170631); +tree.insert([0, 0, 0, 0, 0, 0, 0, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170633); +tree.insert([0, 0, 0, 0, 0, 0, 0, 242], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170634); +tree.insert([0, 0, 0, 0, 0, 0, 0, 122], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170635); +tree.insert([0, 0, 0, 0, 0, 0, 0, 153], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170636); +tree.insert([0, 0, 0, 0, 0, 0, 1, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170638); +tree.insert([0, 0, 0, 0, 0, 0, 0, 182], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170639); +tree.insert([0, 0, 0, 0, 0, 0, 1, 213], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170641); +tree.insert([0, 0, 0, 0, 0, 0, 2, 150], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170643); +tree.insert([0, 0, 0, 0, 0, 0, 0, 118], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170644); +tree.insert([0, 0, 0, 0, 0, 0, 1, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170645); +tree.insert([0, 0, 0, 0, 0, 0, 0, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170647); +tree.insert([0, 0, 0, 0, 0, 0, 0, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170648); +tree.insert([0, 0, 0, 0, 0, 0, 0, 73], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170649); +tree.insert([0, 0, 0, 0, 0, 0, 0, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170650); +tree.insert([0, 0, 0, 0, 0, 0, 1, 1], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170651); +tree.insert([0, 0, 0, 0, 0, 0, 2, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170653); +tree.insert([0, 0, 0, 0, 0, 0, 1, 204], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170654); +tree.insert([0, 0, 0, 0, 0, 0, 1, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170655); +tree.insert([0, 0, 0, 0, 0, 0, 2, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170656); +tree.insert([0, 0, 0, 0, 0, 0, 1, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170657); +tree.insert([0, 0, 0, 0, 0, 0, 2, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170658); +tree.insert([0, 0, 0, 0, 0, 0, 0, 147], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170659); +tree.insert([0, 0, 0, 0, 0, 0, 1, 248], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170660); +tree.insert([0, 0, 0, 0, 0, 0, 2, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170661); +tree.insert([0, 0, 0, 0, 0, 0, 1, 113], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170662); +tree.insert([0, 0, 0, 0, 0, 0, 0, 79], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170664); +tree.insert([0, 0, 0, 0, 0, 0, 1, 216], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170665); +tree.insert([0, 0, 0, 0, 0, 0, 0, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170666); +tree.insert([0, 0, 0, 0, 0, 0, 0, 42], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170669); +tree.insert([0, 0, 0, 0, 0, 0, 1, 14], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170671); +tree.insert([0, 0, 0, 0, 0, 0, 0, 38], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170675); +tree.insert([0, 0, 0, 0, 0, 0, 1, 156], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170676); +tree.insert([0, 0, 0, 0, 0, 0, 0, 123], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170677); +tree.insert([0, 0, 0, 0, 0, 0, 0, 238], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170679); +tree.insert([0, 0, 0, 0, 0, 0, 0, 164], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170680); +tree.insert([0, 0, 0, 0, 0, 0, 1, 244], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170681); +tree.insert([0, 0, 0, 0, 0, 0, 0, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170687); +tree.insert([0, 0, 0, 0, 0, 0, 0, 201], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170689); +tree.insert([0, 0, 0, 0, 0, 0, 1, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170690); +tree.insert([0, 0, 0, 0, 0, 0, 0, 247], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170692); +tree.insert([0, 0, 0, 0, 0, 0, 1, 29], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170694); +tree.insert([0, 0, 0, 0, 0, 0, 1, 195], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170695); +tree.insert([0, 0, 0, 0, 0, 0, 1, 158], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170699); +tree.insert([0, 0, 0, 0, 0, 0, 0, 236], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170702); +tree.insert([0, 0, 0, 0, 0, 0, 2, 11], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170705); +tree.insert([0, 0, 0, 0, 0, 0, 0, 107], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170707); +tree.insert([0, 0, 0, 0, 0, 0, 1, 167], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170709); +tree.insert([0, 0, 0, 0, 0, 0, 0, 180], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170710); +tree.insert([0, 0, 0, 0, 0, 0, 3, 171], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170711); +tree.insert([0, 0, 0, 0, 0, 0, 3, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170712); +tree.insert([0, 0, 0, 0, 0, 0, 1, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170713); +tree.insert([0, 0, 0, 0, 0, 0, 3, 66], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170714); +tree.insert([0, 0, 0, 0, 0, 0, 1, 90], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170715); +tree.insert([0, 0, 0, 0, 0, 0, 1, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170716); +tree.insert([0, 0, 0, 0, 0, 0, 1, 166], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170717); +tree.insert([0, 0, 0, 0, 0, 0, 3, 222], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170718); +tree.insert([0, 0, 0, 0, 0, 0, 0, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170719); +tree.insert([0, 0, 0, 0, 0, 0, 3, 209], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170720); +tree.insert([0, 0, 0, 0, 0, 0, 1, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170721); +tree.insert([0, 0, 0, 0, 0, 0, 3, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170722); +tree.insert([0, 0, 0, 0, 0, 0, 0, 117], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170723); +tree.insert([0, 0, 0, 0, 0, 0, 2, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170724); +tree.insert([0, 0, 0, 0, 0, 0, 3, 44], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170725); +tree.insert([0, 0, 0, 0, 0, 0, 2, 158], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170726); +tree.insert([0, 0, 0, 0, 0, 0, 3, 67], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170727); +tree.insert([0, 0, 0, 0, 0, 0, 0, 152], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170728); +tree.insert([0, 0, 0, 0, 0, 0, 3, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170729); +tree.insert([0, 0, 0, 0, 0, 0, 0, 75], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170730); +tree.insert([0, 0, 0, 0, 0, 0, 1, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170731); +tree.insert([0, 0, 0, 0, 0, 0, 1, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170732); +tree.insert([0, 0, 0, 0, 0, 0, 1, 143], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170733); +tree.insert([0, 0, 0, 0, 0, 0, 1, 254], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170735); +tree.insert([0, 0, 0, 0, 0, 0, 2, 183], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170736); +tree.insert([0, 0, 0, 0, 0, 0, 1, 236], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170737); +tree.insert([0, 0, 0, 0, 0, 0, 3, 13], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170738); +tree.insert([0, 0, 0, 0, 0, 0, 2, 6], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170739); +tree.insert([0, 0, 0, 0, 0, 0, 2, 7], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170740); +tree.insert([0, 0, 0, 0, 0, 0, 2, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170741); +tree.insert([0, 0, 0, 0, 0, 0, 2, 70], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170742); +tree.insert([0, 0, 0, 0, 0, 0, 3, 29], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170743); +tree.insert([0, 0, 0, 0, 0, 0, 0, 101], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170744); +tree.insert([0, 0, 0, 0, 0, 0, 1, 226], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170745); +tree.insert([0, 0, 0, 0, 0, 0, 3, 182], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170746); +tree.insert([0, 0, 0, 0, 0, 0, 0, 120], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170747); +tree.insert([0, 0, 0, 0, 0, 0, 1, 215], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170748); +tree.insert([0, 0, 0, 0, 0, 0, 1, 3], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170749); +tree.insert([0, 0, 0, 0, 0, 0, 3, 65], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170751); +tree.insert([0, 0, 0, 0, 0, 0, 1, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170752); +tree.insert([0, 0, 0, 0, 0, 0, 3, 115], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170754); +tree.insert([0, 0, 0, 0, 0, 0, 0, 102], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170755); +tree.insert([0, 0, 0, 0, 0, 0, 0, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170756); +tree.insert([0, 0, 0, 0, 0, 0, 1, 100], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170757); +tree.insert([0, 0, 0, 0, 0, 0, 2, 134], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170758); +tree.insert([0, 0, 0, 0, 0, 0, 1, 86], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170759); +tree.insert([0, 0, 0, 0, 0, 0, 0, 95], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170760); +tree.insert([0, 0, 0, 0, 0, 0, 1, 176], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170761); +tree.insert([0, 0, 0, 0, 0, 0, 2, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170764); +tree.insert([0, 0, 0, 0, 0, 0, 3, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170765); +tree.insert([0, 0, 0, 0, 0, 0, 3, 124], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170766); +tree.insert([0, 0, 0, 0, 0, 0, 1, 11], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170767); +tree.insert([0, 0, 0, 0, 0, 0, 2, 248], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170768); +tree.insert([0, 0, 0, 0, 0, 0, 1, 129], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170769); +tree.insert([0, 0, 0, 0, 0, 0, 0, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170770); +tree.insert([0, 0, 0, 0, 0, 0, 1, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170771); +tree.insert([0, 0, 0, 0, 0, 0, 1, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170772); +tree.insert([0, 0, 0, 0, 0, 0, 3, 210], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170773); +tree.insert([0, 0, 0, 0, 0, 0, 1, 81], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170774); +tree.insert([0, 0, 0, 0, 0, 0, 0, 125], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170775); +tree.insert([0, 0, 0, 0, 0, 0, 1, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170776); +tree.insert([0, 0, 0, 0, 0, 0, 1, 186], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170777); +tree.insert([0, 0, 0, 0, 0, 0, 3, 75], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170778); +tree.insert([0, 0, 0, 0, 0, 0, 3, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170779); +tree.insert([0, 0, 0, 0, 0, 0, 2, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170780); +tree.insert([0, 0, 0, 0, 0, 0, 3, 46], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170781); +tree.insert([0, 0, 0, 0, 0, 0, 1, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170782); +tree.insert([0, 0, 0, 0, 0, 0, 0, 144], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170784); +tree.insert([0, 0, 0, 0, 0, 0, 2, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170785); +tree.insert([0, 0, 0, 0, 0, 0, 2, 236], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170786); +tree.insert([0, 0, 0, 0, 0, 0, 1, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170787); +tree.insert([0, 0, 0, 0, 0, 0, 3, 18], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170788); +tree.insert([0, 0, 0, 0, 0, 0, 3, 175], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170789); +tree.insert([0, 0, 0, 0, 0, 0, 1, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170790); +tree.insert([0, 0, 0, 0, 0, 0, 2, 255], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170791); +tree.insert([0, 0, 0, 0, 0, 0, 0, 109], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170793); +tree.insert([0, 0, 0, 0, 0, 0, 1, 148], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170794); +tree.insert([0, 0, 0, 0, 0, 0, 0, 155], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170795); +tree.insert([0, 0, 0, 0, 0, 0, 2, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170796); +tree.insert([0, 0, 0, 0, 0, 0, 3, 91], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170797); +tree.insert([0, 0, 0, 0, 0, 0, 0, 199], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170798); +tree.insert([0, 0, 0, 0, 0, 0, 1, 247], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170799); +tree.flush_active_memtable(170701)?; +tree.insert([0, 0, 0, 0, 0, 0, 1, 192], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170801); +tree.insert([0, 0, 0, 0, 0, 0, 1, 165], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170802); +tree.insert([0, 0, 0, 0, 0, 0, 0, 116], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170804); +tree.insert([0, 0, 0, 0, 0, 0, 0, 32], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170805); +tree.insert([0, 0, 0, 0, 0, 0, 0, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170809); +tree.insert([0, 0, 0, 0, 0, 0, 1, 124], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170812); +tree.insert([0, 0, 0, 0, 0, 0, 2, 145], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170813); +tree.insert([0, 0, 0, 0, 0, 0, 1, 251], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170814); +tree.insert([0, 0, 0, 0, 0, 0, 0, 138], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170815); +tree.insert([0, 0, 0, 0, 0, 0, 3, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170816); +tree.insert([0, 0, 0, 0, 0, 0, 3, 189], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170817); +tree.insert([0, 0, 0, 0, 0, 0, 2, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170818); +tree.insert([0, 0, 0, 0, 0, 0, 3, 145], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170819); +tree.insert([0, 0, 0, 0, 0, 0, 3, 43], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170820); +tree.insert([0, 0, 0, 0, 0, 0, 0, 5], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170822); +tree.insert([0, 0, 0, 0, 0, 0, 2, 153], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170823); +tree.insert([0, 0, 0, 0, 0, 0, 0, 211], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170824); +tree.insert([0, 0, 0, 0, 0, 0, 3, 92], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170825); +tree.insert([0, 0, 0, 0, 0, 0, 0, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170826); +tree.insert([0, 0, 0, 0, 0, 0, 2, 171], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170827); +tree.insert([0, 0, 0, 0, 0, 0, 0, 133], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170828); +tree.insert([0, 0, 0, 0, 0, 0, 2, 167], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170829); +tree.insert([0, 0, 0, 0, 0, 0, 0, 226], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170830); +tree.insert([0, 0, 0, 0, 0, 0, 1, 242], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170831); +tree.insert([0, 0, 0, 0, 0, 0, 0, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170832); +tree.insert([0, 0, 0, 0, 0, 0, 1, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170833); +tree.insert([0, 0, 0, 0, 0, 0, 1, 240], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170834); +tree.insert([0, 0, 0, 0, 0, 0, 2, 84], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170836); +tree.insert([0, 0, 0, 0, 0, 0, 3, 185], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170837); +tree.insert([0, 0, 0, 0, 0, 0, 0, 145], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170838); +tree.insert([0, 0, 0, 0, 0, 0, 2, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170840); +tree.insert([0, 0, 0, 0, 0, 0, 3, 143], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170841); +tree.insert([0, 0, 0, 0, 0, 0, 0, 227], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170842); +tree.insert([0, 0, 0, 0, 0, 0, 2, 5], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170843); +tree.insert([0, 0, 0, 0, 0, 0, 1, 151], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170844); +tree.insert([0, 0, 0, 0, 0, 0, 1, 118], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170846); +tree.insert([0, 0, 0, 0, 0, 0, 2, 240], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170847); +tree.insert([0, 0, 0, 0, 0, 0, 3, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170848); +tree.insert([0, 0, 0, 0, 0, 0, 1, 178], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170850); +tree.insert([0, 0, 0, 0, 0, 0, 0, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170851); +tree.insert([0, 0, 0, 0, 0, 0, 1, 101], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170852); +tree.insert([0, 0, 0, 0, 0, 0, 3, 199], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170853); +tree.insert([0, 0, 0, 0, 0, 0, 2, 76], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170855); +tree.insert([0, 0, 0, 0, 0, 0, 1, 234], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170856); +tree.insert([0, 0, 0, 0, 0, 0, 1, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170857); +tree.insert([0, 0, 0, 0, 0, 0, 0, 43], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170859); +tree.insert([0, 0, 0, 0, 0, 0, 1, 228], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170860); +tree.insert([0, 0, 0, 0, 0, 0, 2, 159], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170861); +tree.insert([0, 0, 0, 0, 0, 0, 2, 233], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170862); +tree.insert([0, 0, 0, 0, 0, 0, 3, 69], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170863); +tree.insert([0, 0, 0, 0, 0, 0, 1, 95], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170864); +tree.insert([0, 0, 0, 0, 0, 0, 0, 254], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170865); +tree.insert([0, 0, 0, 0, 0, 0, 1, 5], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170866); +tree.insert([0, 0, 0, 0, 0, 0, 2, 61], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170868); +tree.insert([0, 0, 0, 0, 0, 0, 3, 4], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170869); +tree.insert([0, 0, 0, 0, 0, 0, 3, 151], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170871); +tree.insert([0, 0, 0, 0, 0, 0, 0, 239], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170873); +tree.insert([0, 0, 0, 0, 0, 0, 0, 113], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170874); +tree.insert([0, 0, 0, 0, 0, 0, 1, 137], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170875); +tree.insert([0, 0, 0, 0, 0, 0, 2, 36], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170877); +tree.insert([0, 0, 0, 0, 0, 0, 2, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170879); +tree.insert([0, 0, 0, 0, 0, 0, 2, 230], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170880); +tree.insert([0, 0, 0, 0, 0, 0, 3, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170881); +tree.insert([0, 0, 0, 0, 0, 0, 3, 137], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170882); +tree.insert([0, 0, 0, 0, 0, 0, 2, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170883); +tree.insert([0, 0, 0, 0, 0, 0, 0, 229], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170885); +tree.insert([0, 0, 0, 0, 0, 0, 2, 182], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170886); +tree.insert([0, 0, 0, 0, 0, 0, 0, 31], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170887); +tree.insert([0, 0, 0, 0, 0, 0, 0, 13], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170888); +tree.insert([0, 0, 0, 0, 0, 0, 2, 91], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170889); +tree.insert([0, 0, 0, 0, 0, 0, 3, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170890); +tree.insert([0, 0, 0, 0, 0, 0, 1, 222], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170891); +tree.insert([0, 0, 0, 0, 0, 0, 1, 136], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170892); +tree.insert([0, 0, 0, 0, 0, 0, 2, 44], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170894); +tree.insert([0, 0, 0, 0, 0, 0, 1, 180], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170896); +tree.insert([0, 0, 0, 0, 0, 0, 0, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170897); +tree.insert([0, 0, 0, 0, 0, 0, 2, 107], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170898); +tree.insert([0, 0, 0, 0, 0, 0, 3, 27], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170899); +tree.insert([0, 0, 0, 0, 0, 0, 3, 135], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170900); +tree.insert([0, 0, 0, 0, 0, 0, 2, 251], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170901); +tree.insert([0, 0, 0, 0, 0, 0, 0, 167], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170902); +tree.insert([0, 0, 0, 0, 0, 0, 3, 32], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170903); +tree.insert([0, 0, 0, 0, 0, 0, 1, 104], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170904); +tree.insert([0, 0, 0, 0, 0, 0, 3, 204], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170905); +tree.insert([0, 0, 0, 0, 0, 0, 1, 232], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170906); +tree.insert([0, 0, 0, 0, 0, 0, 3, 134], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170907); +tree.insert([0, 0, 0, 0, 0, 0, 0, 119], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170908); +tree.insert([0, 0, 0, 0, 0, 0, 2, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170909); +tree.insert([0, 0, 0, 0, 0, 0, 3, 117], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170911); +tree.insert([0, 0, 0, 0, 0, 0, 3, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170914); +tree.insert([0, 0, 0, 0, 0, 0, 2, 97], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170915); +tree.insert([0, 0, 0, 0, 0, 0, 3, 20], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170918); +tree.insert([0, 0, 0, 0, 0, 0, 0, 231], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170919); +tree.insert([0, 0, 0, 0, 0, 0, 2, 213], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170920); +tree.insert([0, 0, 0, 0, 0, 0, 1, 141], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170921); +tree.insert([0, 0, 0, 0, 0, 0, 0, 128], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170922); +tree.insert([0, 0, 0, 0, 0, 0, 0, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170923); +tree.insert([0, 0, 0, 0, 0, 0, 2, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170924); +tree.insert([0, 0, 0, 0, 0, 0, 1, 40], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170926); +tree.insert([0, 0, 0, 0, 0, 0, 3, 173], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170927); +tree.insert([0, 0, 0, 0, 0, 0, 3, 106], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170928); +tree.insert([0, 0, 0, 0, 0, 0, 2, 59], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170929); +tree.insert([0, 0, 0, 0, 0, 0, 1, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170930); +tree.insert([0, 0, 0, 0, 0, 0, 1, 205], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170932); +tree.insert([0, 0, 0, 0, 0, 0, 0, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170933); +tree.insert([0, 0, 0, 0, 0, 0, 1, 51], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170935); +tree.insert([0, 0, 0, 0, 0, 0, 3, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170937); +tree.insert([0, 0, 0, 0, 0, 0, 1, 246], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170938); +tree.insert([0, 0, 0, 0, 0, 0, 3, 186], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170939); +tree.insert([0, 0, 0, 0, 0, 0, 1, 160], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170940); +tree.insert([0, 0, 0, 0, 0, 0, 1, 115], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170943); +tree.insert([0, 0, 0, 0, 0, 0, 1, 194], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170945); +tree.insert([0, 0, 0, 0, 0, 0, 0, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170946); +tree.insert([0, 0, 0, 0, 0, 0, 2, 204], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170947); +tree.insert([0, 0, 0, 0, 0, 0, 2, 216], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170949); +tree.insert([0, 0, 0, 0, 0, 0, 2, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170953); +tree.insert([0, 0, 0, 0, 0, 0, 2, 94], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170956); +tree.insert([0, 0, 0, 0, 0, 0, 1, 128], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170957); +tree.insert([0, 0, 0, 0, 0, 0, 1, 15], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170958); +tree.insert([0, 0, 0, 0, 0, 0, 1, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170959); +tree.insert([0, 0, 0, 0, 0, 0, 1, 99], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170961); +tree.insert([0, 0, 0, 0, 0, 0, 0, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170964); +tree.insert([0, 0, 0, 0, 0, 0, 0, 150], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170965); +tree.insert([0, 0, 0, 0, 0, 0, 1, 52], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170966); +tree.insert([0, 0, 0, 0, 0, 0, 3, 120], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170967); +tree.insert([0, 0, 0, 0, 0, 0, 1, 57], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170968); +tree.insert([0, 0, 0, 0, 0, 0, 2, 188], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170970); +tree.insert([0, 0, 0, 0, 0, 0, 3, 23], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170972); +tree.insert([0, 0, 0, 0, 0, 0, 3, 193], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170973); +tree.insert([0, 0, 0, 0, 0, 0, 3, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170974); +tree.insert([0, 0, 0, 0, 0, 0, 2, 212], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170976); +tree.insert([0, 0, 0, 0, 0, 0, 3, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170978); +tree.insert([0, 0, 0, 0, 0, 0, 1, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170980); +tree.insert([0, 0, 0, 0, 0, 0, 3, 52], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170981); +tree.insert([0, 0, 0, 0, 0, 0, 1, 214], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170982); +tree.insert([0, 0, 0, 0, 0, 0, 0, 19], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170983); +tree.insert([0, 0, 0, 0, 0, 0, 0, 246], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170985); +tree.insert([0, 0, 0, 0, 0, 0, 0, 77], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170986); +tree.insert([0, 0, 0, 0, 0, 0, 3, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170987); +tree.insert([0, 0, 0, 0, 0, 0, 2, 162], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170988); +tree.insert([0, 0, 0, 0, 0, 0, 0, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170989); +tree.insert([0, 0, 0, 0, 0, 0, 2, 127], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170990); +tree.insert([0, 0, 0, 0, 0, 0, 0, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170991); +tree.insert([0, 0, 0, 0, 0, 0, 1, 173], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170993); +tree.insert([0, 0, 0, 0, 0, 0, 3, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170994); +tree.insert([0, 0, 0, 0, 0, 0, 0, 192], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170995); +tree.insert([0, 0, 0, 0, 0, 0, 2, 43], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170996); +tree.insert([0, 0, 0, 0, 0, 0, 1, 38], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170998); +tree.insert([0, 0, 0, 0, 0, 0, 1, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 170999); +tree.insert([0, 0, 0, 0, 0, 0, 0, 90], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171000); +tree.flush_active_memtable(170901)?; +tree.insert([0, 0, 0, 0, 0, 0, 2, 163], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171001); +tree.insert([0, 0, 0, 0, 0, 0, 2, 195], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171003); +tree.insert([0, 0, 0, 0, 0, 0, 3, 86], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171004); +tree.insert([0, 0, 0, 0, 0, 0, 1, 8], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171005); +tree.insert([0, 0, 0, 0, 0, 0, 0, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171006); +tree.insert([0, 0, 0, 0, 0, 0, 3, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171007); +tree.insert([0, 0, 0, 0, 0, 0, 1, 208], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171008); +tree.insert([0, 0, 0, 0, 0, 0, 0, 237], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171010); +tree.insert([0, 0, 0, 0, 0, 0, 1, 190], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171012); +tree.insert([0, 0, 0, 0, 0, 0, 2, 221], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171013); +tree.insert([0, 0, 0, 0, 0, 0, 2, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171014); +tree.insert([0, 0, 0, 0, 0, 0, 1, 231], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171015); +tree.insert([0, 0, 0, 0, 0, 0, 0, 97], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171016); +tree.insert([0, 0, 0, 0, 0, 0, 1, 217], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171017); +tree.insert([0, 0, 0, 0, 0, 0, 2, 224], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171018); +tree.insert([0, 0, 0, 0, 0, 0, 0, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171019); +tree.insert([0, 0, 0, 0, 0, 0, 3, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171022); +tree.insert([0, 0, 0, 0, 0, 0, 0, 235], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171024); +tree.insert([0, 0, 0, 0, 0, 0, 1, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171027); +tree.insert([0, 0, 0, 0, 0, 0, 1, 107], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171034); +tree.insert([0, 0, 0, 0, 0, 0, 1, 233], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171042); +tree.insert([0, 0, 0, 0, 0, 0, 1, 0], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171044); +tree.insert([0, 0, 0, 0, 0, 0, 2, 77], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171045); +tree.insert([0, 0, 0, 0, 0, 0, 0, 253], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171050); +tree.insert([0, 0, 0, 0, 0, 0, 0, 241], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171051); +tree.insert([0, 0, 0, 0, 0, 0, 2, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171052); +tree.insert([0, 0, 0, 0, 0, 0, 0, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171053); +tree.insert([0, 0, 0, 0, 0, 0, 0, 24], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171063); +tree.insert([0, 0, 0, 0, 0, 0, 1, 24], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171065); +tree.insert([0, 0, 0, 0, 0, 0, 3, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171070); +tree.insert([0, 0, 0, 0, 0, 0, 3, 138], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171071); +tree.insert([0, 0, 0, 0, 0, 0, 3, 126], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171075); +tree.insert([0, 0, 0, 0, 0, 0, 3, 42], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171076); +tree.insert([0, 0, 0, 0, 0, 0, 1, 146], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171078); +tree.insert([0, 0, 0, 0, 0, 0, 3, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171080); +tree.insert([0, 0, 0, 0, 0, 0, 0, 233], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171082); +tree.insert([0, 0, 0, 0, 0, 0, 0, 1], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171085); +tree.insert([0, 0, 0, 0, 0, 0, 2, 42], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171088); +tree.insert([0, 0, 0, 0, 0, 0, 0, 23], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171089); +tree.insert([0, 0, 0, 0, 0, 0, 1, 202], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171091); +tree.insert([0, 0, 0, 0, 0, 0, 1, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171092); +tree.insert([0, 0, 0, 0, 0, 0, 2, 193], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171094); +tree.insert([0, 0, 0, 0, 0, 0, 1, 21], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171095); +tree.insert([0, 0, 0, 0, 0, 0, 1, 37], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171097); +tree.insert([0, 0, 0, 0, 0, 0, 2, 191], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171098); +tree.insert([0, 0, 0, 0, 0, 0, 0, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171099); +tree.insert([0, 0, 0, 0, 0, 0, 2, 186], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171103); +tree.insert([0, 0, 0, 0, 0, 0, 1, 237], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171104); +tree.insert([0, 0, 0, 0, 0, 0, 0, 67], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171105); +tree.insert([0, 0, 0, 0, 0, 0, 0, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171110); +tree.insert([0, 0, 0, 0, 0, 0, 2, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171111); +tree.insert([0, 0, 0, 0, 0, 0, 1, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171113); +tree.insert([0, 0, 0, 0, 0, 0, 2, 119], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171114); +tree.insert([0, 0, 0, 0, 0, 0, 1, 125], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171115); +tree.insert([0, 0, 0, 0, 0, 0, 1, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171116); +tree.insert([0, 0, 0, 0, 0, 0, 0, 166], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171117); +tree.insert([0, 0, 0, 0, 0, 0, 2, 249], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171118); +tree.insert([0, 0, 0, 0, 0, 0, 2, 223], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171122); +tree.insert([0, 0, 0, 0, 0, 0, 0, 108], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171133); +tree.insert([0, 0, 0, 0, 0, 0, 1, 84], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171173); +tree.insert([0, 0, 0, 0, 0, 0, 0, 224], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171174); +tree.insert([0, 0, 0, 0, 0, 0, 1, 152], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171175); +tree.insert([0, 0, 0, 0, 0, 0, 2, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171180); +tree.flush_active_memtable(171101)?; +tree.compact(compaction.clone(), 171101)?; +tree.insert([0, 0, 0, 0, 0, 0, 3, 6], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171201); +tree.insert([0, 0, 0, 0, 0, 0, 1, 217], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171202); +tree.insert([0, 0, 0, 0, 0, 0, 3, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171203); +tree.insert([0, 0, 0, 0, 0, 0, 2, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171204); +tree.insert([0, 0, 0, 0, 0, 0, 0, 232], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171205); +tree.insert([0, 0, 0, 0, 0, 0, 3, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171206); +tree.insert([0, 0, 0, 0, 0, 0, 3, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171207); +tree.insert([0, 0, 0, 0, 0, 0, 3, 216], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171208); +tree.insert([0, 0, 0, 0, 0, 0, 1, 161], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171209); +tree.insert([0, 0, 0, 0, 0, 0, 0, 106], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171210); +tree.insert([0, 0, 0, 0, 0, 0, 0, 6], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171211); +tree.insert([0, 0, 0, 0, 0, 0, 2, 69], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171212); +tree.insert([0, 0, 0, 0, 0, 0, 2, 229], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171213); +tree.insert([0, 0, 0, 0, 0, 0, 0, 210], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171214); +tree.insert([0, 0, 0, 0, 0, 0, 0, 65], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171215); +tree.insert([0, 0, 0, 0, 0, 0, 0, 120], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171216); +tree.insert([0, 0, 0, 0, 0, 0, 1, 203], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171217); +tree.insert([0, 0, 0, 0, 0, 0, 2, 194], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171218); +tree.insert([0, 0, 0, 0, 0, 0, 0, 209], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171219); +tree.insert([0, 0, 0, 0, 0, 0, 0, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171220); +tree.insert([0, 0, 0, 0, 0, 0, 3, 162], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171221); +tree.insert([0, 0, 0, 0, 0, 0, 1, 46], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171223); +tree.insert([0, 0, 0, 0, 0, 0, 0, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171224); +tree.insert([0, 0, 0, 0, 0, 0, 1, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171225); +tree.insert([0, 0, 0, 0, 0, 0, 0, 37], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171226); +tree.insert([0, 0, 0, 0, 0, 0, 2, 200], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171227); +tree.insert([0, 0, 0, 0, 0, 0, 3, 200], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171228); +tree.insert([0, 0, 0, 0, 0, 0, 2, 46], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171229); +tree.insert([0, 0, 0, 0, 0, 0, 1, 21], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171230); +tree.insert([0, 0, 0, 0, 0, 0, 2, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171231); +tree.insert([0, 0, 0, 0, 0, 0, 2, 187], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171232); +tree.insert([0, 0, 0, 0, 0, 0, 2, 235], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171233); +tree.insert([0, 0, 0, 0, 0, 0, 0, 178], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171234); +tree.insert([0, 0, 0, 0, 0, 0, 0, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171235); +tree.insert([0, 0, 0, 0, 0, 0, 1, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171236); +tree.insert([0, 0, 0, 0, 0, 0, 1, 171], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171237); +tree.insert([0, 0, 0, 0, 0, 0, 1, 242], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171238); +tree.insert([0, 0, 0, 0, 0, 0, 0, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171240); +tree.insert([0, 0, 0, 0, 0, 0, 1, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171241); +tree.insert([0, 0, 0, 0, 0, 0, 3, 143], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171242); +tree.insert([0, 0, 0, 0, 0, 0, 0, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171243); +tree.insert([0, 0, 0, 0, 0, 0, 3, 37], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171244); +tree.insert([0, 0, 0, 0, 0, 0, 0, 187], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171245); +tree.insert([0, 0, 0, 0, 0, 0, 2, 226], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171246); +tree.insert([0, 0, 0, 0, 0, 0, 3, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171247); +tree.insert([0, 0, 0, 0, 0, 0, 1, 135], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171248); +tree.insert([0, 0, 0, 0, 0, 0, 2, 103], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171249); +tree.insert([0, 0, 0, 0, 0, 0, 1, 3], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171250); +tree.insert([0, 0, 0, 0, 0, 0, 1, 245], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171251); +tree.insert([0, 0, 0, 0, 0, 0, 1, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171252); +tree.insert([0, 0, 0, 0, 0, 0, 0, 206], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171253); +tree.insert([0, 0, 0, 0, 0, 0, 0, 195], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171254); +tree.insert([0, 0, 0, 0, 0, 0, 3, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171255); +tree.insert([0, 0, 0, 0, 0, 0, 2, 153], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171256); +tree.insert([0, 0, 0, 0, 0, 0, 3, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171257); +tree.insert([0, 0, 0, 0, 0, 0, 0, 140], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171258); +tree.insert([0, 0, 0, 0, 0, 0, 2, 59], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171259); +tree.insert([0, 0, 0, 0, 0, 0, 2, 171], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171260); +tree.insert([0, 0, 0, 0, 0, 0, 0, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171262); +tree.insert([0, 0, 0, 0, 0, 0, 1, 184], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171263); +tree.insert([0, 0, 0, 0, 0, 0, 3, 187], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171264); +tree.insert([0, 0, 0, 0, 0, 0, 0, 161], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171265); +tree.insert([0, 0, 0, 0, 0, 0, 2, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171266); +tree.insert([0, 0, 0, 0, 0, 0, 2, 21], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171267); +tree.insert([0, 0, 0, 0, 0, 0, 1, 182], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171268); +tree.insert([0, 0, 0, 0, 0, 0, 3, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171269); +tree.insert([0, 0, 0, 0, 0, 0, 3, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171270); +tree.insert([0, 0, 0, 0, 0, 0, 1, 241], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171271); +tree.insert([0, 0, 0, 0, 0, 0, 1, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171272); +tree.insert([0, 0, 0, 0, 0, 0, 2, 104], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171273); +tree.insert([0, 0, 0, 0, 0, 0, 0, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171274); +tree.insert([0, 0, 0, 0, 0, 0, 3, 23], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171275); +tree.insert([0, 0, 0, 0, 0, 0, 0, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171276); +tree.insert([0, 0, 0, 0, 0, 0, 0, 101], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171277); +tree.insert([0, 0, 0, 0, 0, 0, 0, 250], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171278); +tree.insert([0, 0, 0, 0, 0, 0, 0, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171279); +tree.insert([0, 0, 0, 0, 0, 0, 3, 161], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171280); +tree.insert([0, 0, 0, 0, 0, 0, 1, 122], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171281); +tree.insert([0, 0, 0, 0, 0, 0, 0, 217], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171282); +tree.insert([0, 0, 0, 0, 0, 0, 3, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171283); +tree.insert([0, 0, 0, 0, 0, 0, 0, 9], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171284); +tree.insert([0, 0, 0, 0, 0, 0, 0, 185], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171285); +tree.insert([0, 0, 0, 0, 0, 0, 2, 56], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171286); +tree.insert([0, 0, 0, 0, 0, 0, 0, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171287); +tree.insert([0, 0, 0, 0, 0, 0, 1, 216], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171288); +tree.insert([0, 0, 0, 0, 0, 0, 2, 185], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171289); +tree.insert([0, 0, 0, 0, 0, 0, 0, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171290); +tree.insert([0, 0, 0, 0, 0, 0, 2, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171291); +tree.insert([0, 0, 0, 0, 0, 0, 1, 236], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171292); +tree.insert([0, 0, 0, 0, 0, 0, 0, 40], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171293); +tree.insert([0, 0, 0, 0, 0, 0, 3, 123], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171294); +tree.insert([0, 0, 0, 0, 0, 0, 0, 238], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171295); +tree.insert([0, 0, 0, 0, 0, 0, 0, 171], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171296); +tree.insert([0, 0, 0, 0, 0, 0, 0, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171297); +tree.insert([0, 0, 0, 0, 0, 0, 3, 81], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171298); +tree.insert([0, 0, 0, 0, 0, 0, 0, 76], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171300); +tree.insert([0, 0, 0, 0, 0, 0, 2, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171301); +tree.insert([0, 0, 0, 0, 0, 0, 3, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171302); +tree.insert([0, 0, 0, 0, 0, 0, 3, 217], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171303); +tree.insert([0, 0, 0, 0, 0, 0, 2, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171304); +tree.insert([0, 0, 0, 0, 0, 0, 0, 218], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171305); +tree.insert([0, 0, 0, 0, 0, 0, 3, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171306); +tree.insert([0, 0, 0, 0, 0, 0, 3, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171307); +tree.insert([0, 0, 0, 0, 0, 0, 2, 94], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171308); +tree.insert([0, 0, 0, 0, 0, 0, 0, 207], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171309); +tree.insert([0, 0, 0, 0, 0, 0, 0, 92], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171310); +tree.insert([0, 0, 0, 0, 0, 0, 0, 143], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171311); +tree.insert([0, 0, 0, 0, 0, 0, 0, 155], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171312); +tree.insert([0, 0, 0, 0, 0, 0, 1, 168], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171313); +tree.insert([0, 0, 0, 0, 0, 0, 1, 232], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171314); +tree.insert([0, 0, 0, 0, 0, 0, 2, 122], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171315); +tree.insert([0, 0, 0, 0, 0, 0, 1, 108], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171316); +tree.insert([0, 0, 0, 0, 0, 0, 3, 126], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171317); +tree.insert([0, 0, 0, 0, 0, 0, 0, 153], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171318); +tree.insert([0, 0, 0, 0, 0, 0, 3, 106], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171320); +tree.insert([0, 0, 0, 0, 0, 0, 2, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171321); +tree.insert([0, 0, 0, 0, 0, 0, 3, 106], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171322); +tree.insert([0, 0, 0, 0, 0, 0, 0, 91], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171323); +tree.insert([0, 0, 0, 0, 0, 0, 3, 102], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171324); +tree.insert([0, 0, 0, 0, 0, 0, 0, 73], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171325); +tree.insert([0, 0, 0, 0, 0, 0, 3, 157], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171326); +tree.insert([0, 0, 0, 0, 0, 0, 0, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171327); +tree.insert([0, 0, 0, 0, 0, 0, 3, 14], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171328); +tree.insert([0, 0, 0, 0, 0, 0, 1, 59], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171329); +tree.insert([0, 0, 0, 0, 0, 0, 1, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171330); +tree.insert([0, 0, 0, 0, 0, 0, 1, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171331); +tree.insert([0, 0, 0, 0, 0, 0, 0, 69], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171332); +tree.insert([0, 0, 0, 0, 0, 0, 0, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171333); +tree.insert([0, 0, 0, 0, 0, 0, 3, 136], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171334); +tree.insert([0, 0, 0, 0, 0, 0, 2, 61], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171336); +tree.insert([0, 0, 0, 0, 0, 0, 1, 251], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171337); +tree.insert([0, 0, 0, 0, 0, 0, 1, 243], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171338); +tree.insert([0, 0, 0, 0, 0, 0, 1, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171339); +tree.insert([0, 0, 0, 0, 0, 0, 0, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171340); +tree.insert([0, 0, 0, 0, 0, 0, 3, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171341); +tree.insert([0, 0, 0, 0, 0, 0, 2, 139], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171343); +tree.insert([0, 0, 0, 0, 0, 0, 2, 162], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171344); +tree.insert([0, 0, 0, 0, 0, 0, 3, 182], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171345); +tree.insert([0, 0, 0, 0, 0, 0, 3, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171346); +tree.insert([0, 0, 0, 0, 0, 0, 1, 235], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171347); +tree.insert([0, 0, 0, 0, 0, 0, 0, 245], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171348); +tree.insert([0, 0, 0, 0, 0, 0, 2, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171349); +tree.insert([0, 0, 0, 0, 0, 0, 1, 24], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171350); +tree.insert([0, 0, 0, 0, 0, 0, 2, 253], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171351); +tree.insert([0, 0, 0, 0, 0, 0, 3, 75], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171352); +tree.insert([0, 0, 0, 0, 0, 0, 2, 177], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171353); +tree.insert([0, 0, 0, 0, 0, 0, 1, 115], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171354); +tree.insert([0, 0, 0, 0, 0, 0, 3, 186], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171355); +tree.insert([0, 0, 0, 0, 0, 0, 2, 147], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171356); +tree.insert([0, 0, 0, 0, 0, 0, 1, 61], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171357); +tree.insert([0, 0, 0, 0, 0, 0, 0, 251], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171358); +tree.insert([0, 0, 0, 0, 0, 0, 2, 221], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171359); +tree.insert([0, 0, 0, 0, 0, 0, 1, 147], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171360); +tree.insert([0, 0, 0, 0, 0, 0, 3, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171361); +tree.insert([0, 0, 0, 0, 0, 0, 3, 226], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171362); +tree.insert([0, 0, 0, 0, 0, 0, 1, 31], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171363); +tree.insert([0, 0, 0, 0, 0, 0, 2, 155], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171364); +tree.insert([0, 0, 0, 0, 0, 0, 2, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171365); +tree.insert([0, 0, 0, 0, 0, 0, 2, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171366); +tree.insert([0, 0, 0, 0, 0, 0, 2, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171367); +tree.insert([0, 0, 0, 0, 0, 0, 2, 88], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171368); +tree.insert([0, 0, 0, 0, 0, 0, 0, 199], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171369); +tree.insert([0, 0, 0, 0, 0, 0, 1, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171370); +tree.insert([0, 0, 0, 0, 0, 0, 2, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171371); +tree.insert([0, 0, 0, 0, 0, 0, 3, 97], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171372); +tree.insert([0, 0, 0, 0, 0, 0, 1, 251], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171373); +tree.insert([0, 0, 0, 0, 0, 0, 2, 207], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171374); +tree.insert([0, 0, 0, 0, 0, 0, 1, 127], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171375); +tree.insert([0, 0, 0, 0, 0, 0, 0, 193], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171376); +tree.insert([0, 0, 0, 0, 0, 0, 3, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171377); +tree.insert([0, 0, 0, 0, 0, 0, 3, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171378); +tree.insert([0, 0, 0, 0, 0, 0, 2, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171379); +tree.insert([0, 0, 0, 0, 0, 0, 3, 176], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171380); +tree.insert([0, 0, 0, 0, 0, 0, 3, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171381); +tree.insert([0, 0, 0, 0, 0, 0, 2, 230], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171382); +tree.insert([0, 0, 0, 0, 0, 0, 3, 31], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171383); +tree.insert([0, 0, 0, 0, 0, 0, 0, 129], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171385); +tree.insert([0, 0, 0, 0, 0, 0, 2, 201], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171387); +tree.insert([0, 0, 0, 0, 0, 0, 0, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171388); +tree.insert([0, 0, 0, 0, 0, 0, 0, 192], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171389); +tree.insert([0, 0, 0, 0, 0, 0, 1, 210], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171390); +tree.insert([0, 0, 0, 0, 0, 0, 3, 92], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171391); +tree.insert([0, 0, 0, 0, 0, 0, 3, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171392); +tree.insert([0, 0, 0, 0, 0, 0, 2, 91], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171393); +tree.insert([0, 0, 0, 0, 0, 0, 3, 43], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171394); +tree.insert([0, 0, 0, 0, 0, 0, 1, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171395); +tree.insert([0, 0, 0, 0, 0, 0, 2, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171396); +tree.insert([0, 0, 0, 0, 0, 0, 2, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171397); +tree.insert([0, 0, 0, 0, 0, 0, 1, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171398); +tree.insert([0, 0, 0, 0, 0, 0, 1, 128], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171400); +tree.flush_active_memtable(171301)?; +tree.insert([0, 0, 0, 0, 0, 0, 3, 199], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171401); +tree.insert([0, 0, 0, 0, 0, 0, 3, 2], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171402); +tree.insert([0, 0, 0, 0, 0, 0, 1, 139], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171403); +tree.insert([0, 0, 0, 0, 0, 0, 0, 210], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171404); +tree.insert([0, 0, 0, 0, 0, 0, 0, 183], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171405); +tree.insert([0, 0, 0, 0, 0, 0, 0, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171406); +tree.insert([0, 0, 0, 0, 0, 0, 1, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171407); +tree.insert([0, 0, 0, 0, 0, 0, 2, 249], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171408); +tree.insert([0, 0, 0, 0, 0, 0, 3, 27], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171409); +tree.insert([0, 0, 0, 0, 0, 0, 0, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171410); +tree.insert([0, 0, 0, 0, 0, 0, 1, 240], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171411); +tree.insert([0, 0, 0, 0, 0, 0, 0, 224], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171412); +tree.insert([0, 0, 0, 0, 0, 0, 2, 161], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171413); +tree.insert([0, 0, 0, 0, 0, 0, 0, 219], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171414); +tree.insert([0, 0, 0, 0, 0, 0, 1, 200], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171415); +tree.insert([0, 0, 0, 0, 0, 0, 1, 73], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171417); +tree.insert([0, 0, 0, 0, 0, 0, 0, 115], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171418); +tree.insert([0, 0, 0, 0, 0, 0, 0, 119], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171419); +tree.insert([0, 0, 0, 0, 0, 0, 0, 3], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171420); +tree.insert([0, 0, 0, 0, 0, 0, 1, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171421); +tree.insert([0, 0, 0, 0, 0, 0, 0, 40], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171422); +tree.insert([0, 0, 0, 0, 0, 0, 0, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171425); +tree.insert([0, 0, 0, 0, 0, 0, 0, 152], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171426); +tree.insert([0, 0, 0, 0, 0, 0, 1, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171427); +tree.insert([0, 0, 0, 0, 0, 0, 1, 242], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171428); +tree.insert([0, 0, 0, 0, 0, 0, 2, 177], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171431); +tree.insert([0, 0, 0, 0, 0, 0, 1, 87], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171433); +tree.insert([0, 0, 0, 0, 0, 0, 0, 176], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171434); +tree.insert([0, 0, 0, 0, 0, 0, 0, 250], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171436); +tree.insert([0, 0, 0, 0, 0, 0, 1, 15], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171437); +tree.insert([0, 0, 0, 0, 0, 0, 1, 110], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171438); +tree.insert([0, 0, 0, 0, 0, 0, 1, 197], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171439); +tree.insert([0, 0, 0, 0, 0, 0, 0, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171440); +tree.insert([0, 0, 0, 0, 0, 0, 1, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171443); +tree.insert([0, 0, 0, 0, 0, 0, 1, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171444); +tree.insert([0, 0, 0, 0, 0, 0, 3, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171445); +tree.insert([0, 0, 0, 0, 0, 0, 0, 125], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171446); +tree.insert([0, 0, 0, 0, 0, 0, 3, 31], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171449); +tree.insert([0, 0, 0, 0, 0, 0, 3, 90], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171455); +tree.insert([0, 0, 0, 0, 0, 0, 0, 194], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171456); +tree.insert([0, 0, 0, 0, 0, 0, 2, 157], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171457); +tree.insert([0, 0, 0, 0, 0, 0, 2, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171458); +tree.insert([0, 0, 0, 0, 0, 0, 3, 187], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171459); +tree.insert([0, 0, 0, 0, 0, 0, 3, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171460); +tree.insert([0, 0, 0, 0, 0, 0, 0, 163], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171461); +tree.insert([0, 0, 0, 0, 0, 0, 3, 32], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171462); +tree.insert([0, 0, 0, 0, 0, 0, 2, 195], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171463); +tree.insert([0, 0, 0, 0, 0, 0, 0, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171464); +tree.insert([0, 0, 0, 0, 0, 0, 3, 5], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171465); +tree.insert([0, 0, 0, 0, 0, 0, 2, 117], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171466); +tree.insert([0, 0, 0, 0, 0, 0, 3, 88], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171467); +tree.insert([0, 0, 0, 0, 0, 0, 3, 192], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171468); +tree.insert([0, 0, 0, 0, 0, 0, 1, 148], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171469); +tree.insert([0, 0, 0, 0, 0, 0, 3, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171470); +tree.insert([0, 0, 0, 0, 0, 0, 2, 194], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171471); +tree.insert([0, 0, 0, 0, 0, 0, 1, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171472); +tree.insert([0, 0, 0, 0, 0, 0, 2, 14], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171473); +tree.insert([0, 0, 0, 0, 0, 0, 1, 84], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171474); +tree.insert([0, 0, 0, 0, 0, 0, 0, 84], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171475); +tree.insert([0, 0, 0, 0, 0, 0, 2, 225], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171476); +tree.insert([0, 0, 0, 0, 0, 0, 1, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171478); +tree.insert([0, 0, 0, 0, 0, 0, 0, 134], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171479); +tree.insert([0, 0, 0, 0, 0, 0, 3, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171480); +tree.insert([0, 0, 0, 0, 0, 0, 3, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171481); +tree.insert([0, 0, 0, 0, 0, 0, 0, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171482); +tree.insert([0, 0, 0, 0, 0, 0, 3, 61], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171483); +tree.insert([0, 0, 0, 0, 0, 0, 3, 24], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171484); +tree.insert([0, 0, 0, 0, 0, 0, 3, 141], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171485); +tree.insert([0, 0, 0, 0, 0, 0, 2, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171486); +tree.insert([0, 0, 0, 0, 0, 0, 2, 90], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171488); +tree.insert([0, 0, 0, 0, 0, 0, 0, 255], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171489); +tree.insert([0, 0, 0, 0, 0, 0, 0, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171491); +tree.insert([0, 0, 0, 0, 0, 0, 0, 234], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171492); +tree.insert([0, 0, 0, 0, 0, 0, 2, 130], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171493); +tree.insert([0, 0, 0, 0, 0, 0, 0, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171494); +tree.insert([0, 0, 0, 0, 0, 0, 2, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171495); +tree.insert([0, 0, 0, 0, 0, 0, 3, 113], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171496); +tree.insert([0, 0, 0, 0, 0, 0, 3, 38], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171497); +tree.insert([0, 0, 0, 0, 0, 0, 2, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171498); +tree.insert([0, 0, 0, 0, 0, 0, 1, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171499); +tree.insert([0, 0, 0, 0, 0, 0, 0, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171500); +tree.insert([0, 0, 0, 0, 0, 0, 3, 21], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171502); +tree.insert([0, 0, 0, 0, 0, 0, 0, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171503); +tree.insert([0, 0, 0, 0, 0, 0, 3, 132], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171504); +tree.insert([0, 0, 0, 0, 0, 0, 1, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171505); +tree.insert([0, 0, 0, 0, 0, 0, 0, 20], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171506); +tree.insert([0, 0, 0, 0, 0, 0, 1, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171507); +tree.insert([0, 0, 0, 0, 0, 0, 3, 100], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171508); +tree.insert([0, 0, 0, 0, 0, 0, 3, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171509); +tree.insert([0, 0, 0, 0, 0, 0, 0, 242], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171511); +tree.insert([0, 0, 0, 0, 0, 0, 2, 230], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171512); +tree.insert([0, 0, 0, 0, 0, 0, 3, 125], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171514); +tree.insert([0, 0, 0, 0, 0, 0, 1, 239], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171515); +tree.insert([0, 0, 0, 0, 0, 0, 0, 83], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171516); +tree.insert([0, 0, 0, 0, 0, 0, 3, 72], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171517); +tree.insert([0, 0, 0, 0, 0, 0, 1, 108], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171518); +tree.insert([0, 0, 0, 0, 0, 0, 0, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171519); +tree.insert([0, 0, 0, 0, 0, 0, 2, 239], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171520); +tree.insert([0, 0, 0, 0, 0, 0, 2, 3], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171521); +tree.insert([0, 0, 0, 0, 0, 0, 0, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171522); +tree.insert([0, 0, 0, 0, 0, 0, 2, 215], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171524); +tree.insert([0, 0, 0, 0, 0, 0, 2, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171525); +tree.insert([0, 0, 0, 0, 0, 0, 2, 209], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171526); +tree.insert([0, 0, 0, 0, 0, 0, 0, 227], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171527); +tree.insert([0, 0, 0, 0, 0, 0, 1, 116], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171528); +tree.insert([0, 0, 0, 0, 0, 0, 3, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171530); +tree.insert([0, 0, 0, 0, 0, 0, 1, 214], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171531); +tree.insert([0, 0, 0, 0, 0, 0, 3, 177], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171532); +tree.insert([0, 0, 0, 0, 0, 0, 0, 147], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171533); +tree.insert([0, 0, 0, 0, 0, 0, 1, 79], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171534); +tree.insert([0, 0, 0, 0, 0, 0, 0, 186], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171535); +tree.insert([0, 0, 0, 0, 0, 0, 3, 176], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171536); +tree.insert([0, 0, 0, 0, 0, 0, 3, 86], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171537); +tree.insert([0, 0, 0, 0, 0, 0, 3, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171538); +tree.insert([0, 0, 0, 0, 0, 0, 1, 173], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171539); +tree.insert([0, 0, 0, 0, 0, 0, 0, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171540); +tree.insert([0, 0, 0, 0, 0, 0, 1, 157], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171541); +tree.insert([0, 0, 0, 0, 0, 0, 2, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171542); +tree.insert([0, 0, 0, 0, 0, 0, 2, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171545); +tree.insert([0, 0, 0, 0, 0, 0, 0, 254], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171546); +tree.insert([0, 0, 0, 0, 0, 0, 1, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171548); +tree.insert([0, 0, 0, 0, 0, 0, 3, 123], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171550); +tree.insert([0, 0, 0, 0, 0, 0, 2, 158], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171551); +tree.insert([0, 0, 0, 0, 0, 0, 3, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171552); +tree.insert([0, 0, 0, 0, 0, 0, 1, 149], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171553); +tree.insert([0, 0, 0, 0, 0, 0, 3, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171555); +tree.insert([0, 0, 0, 0, 0, 0, 3, 155], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171556); +tree.insert([0, 0, 0, 0, 0, 0, 2, 42], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171557); +tree.insert([0, 0, 0, 0, 0, 0, 2, 21], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171561); +tree.insert([0, 0, 0, 0, 0, 0, 1, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171563); +tree.insert([0, 0, 0, 0, 0, 0, 0, 69], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171564); +tree.insert([0, 0, 0, 0, 0, 0, 0, 193], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171565); +tree.insert([0, 0, 0, 0, 0, 0, 1, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171568); +tree.insert([0, 0, 0, 0, 0, 0, 0, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171570); +tree.insert([0, 0, 0, 0, 0, 0, 1, 178], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171571); +tree.insert([0, 0, 0, 0, 0, 0, 2, 118], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171572); +tree.insert([0, 0, 0, 0, 0, 0, 0, 165], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171573); +tree.insert([0, 0, 0, 0, 0, 0, 0, 141], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171574); +tree.insert([0, 0, 0, 0, 0, 0, 2, 119], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171575); +tree.insert([0, 0, 0, 0, 0, 0, 2, 99], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171576); +tree.insert([0, 0, 0, 0, 0, 0, 1, 250], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171577); +tree.insert([0, 0, 0, 0, 0, 0, 0, 11], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171578); +tree.insert([0, 0, 0, 0, 0, 0, 2, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171581); +tree.insert([0, 0, 0, 0, 0, 0, 1, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171582); +tree.insert([0, 0, 0, 0, 0, 0, 2, 207], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171583); +tree.insert([0, 0, 0, 0, 0, 0, 1, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171584); +tree.insert([0, 0, 0, 0, 0, 0, 1, 231], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171585); +tree.insert([0, 0, 0, 0, 0, 0, 0, 189], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171586); +tree.insert([0, 0, 0, 0, 0, 0, 1, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171589); +tree.insert([0, 0, 0, 0, 0, 0, 2, 92], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171592); +tree.insert([0, 0, 0, 0, 0, 0, 0, 73], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171593); +tree.insert([0, 0, 0, 0, 0, 0, 0, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171595); +tree.insert([0, 0, 0, 0, 0, 0, 0, 52], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171596); +tree.insert([0, 0, 0, 0, 0, 0, 2, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171597); +tree.insert([0, 0, 0, 0, 0, 0, 0, 32], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171599); +tree.flush_active_memtable(171501)?; +tree.insert([0, 0, 0, 0, 0, 0, 1, 224], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171601); +tree.insert([0, 0, 0, 0, 0, 0, 0, 77], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171602); +tree.insert([0, 0, 0, 0, 0, 0, 0, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171604); +tree.insert([0, 0, 0, 0, 0, 0, 1, 94], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171606); +tree.insert([0, 0, 0, 0, 0, 0, 2, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171607); +tree.insert([0, 0, 0, 0, 0, 0, 3, 102], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171608); +tree.insert([0, 0, 0, 0, 0, 0, 2, 11], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171609); +tree.insert([0, 0, 0, 0, 0, 0, 2, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171613); +tree.insert([0, 0, 0, 0, 0, 0, 2, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171614); +tree.insert([0, 0, 0, 0, 0, 0, 1, 199], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171615); +tree.insert([0, 0, 0, 0, 0, 0, 1, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171616); +tree.insert([0, 0, 0, 0, 0, 0, 0, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171618); +tree.insert([0, 0, 0, 0, 0, 0, 0, 249], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171621); +tree.insert([0, 0, 0, 0, 0, 0, 2, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171622); +tree.insert([0, 0, 0, 0, 0, 0, 1, 156], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171623); +tree.insert([0, 0, 0, 0, 0, 0, 2, 73], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171624); +tree.insert([0, 0, 0, 0, 0, 0, 2, 19], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171628); +tree.insert([0, 0, 0, 0, 0, 0, 2, 13], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171629); +tree.insert([0, 0, 0, 0, 0, 0, 2, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171633); +tree.insert([0, 0, 0, 0, 0, 0, 0, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171635); +tree.insert([0, 0, 0, 0, 0, 0, 2, 23], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171636); +tree.insert([0, 0, 0, 0, 0, 0, 3, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171637); +tree.insert([0, 0, 0, 0, 0, 0, 2, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171638); +tree.insert([0, 0, 0, 0, 0, 0, 0, 206], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171639); +tree.insert([0, 0, 0, 0, 0, 0, 0, 29], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171643); +tree.insert([0, 0, 0, 0, 0, 0, 1, 133], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171644); +tree.insert([0, 0, 0, 0, 0, 0, 0, 13], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171645); +tree.insert([0, 0, 0, 0, 0, 0, 1, 120], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171646); +tree.insert([0, 0, 0, 0, 0, 0, 0, 9], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171647); +tree.insert([0, 0, 0, 0, 0, 0, 0, 166], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171649); +tree.insert([0, 0, 0, 0, 0, 0, 1, 168], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171651); +tree.insert([0, 0, 0, 0, 0, 0, 1, 202], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171653); +tree.insert([0, 0, 0, 0, 0, 0, 1, 229], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171657); +tree.insert([0, 0, 0, 0, 0, 0, 2, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171658); +tree.insert([0, 0, 0, 0, 0, 0, 0, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171659); +tree.insert([0, 0, 0, 0, 0, 0, 1, 6], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171662); +tree.insert([0, 0, 0, 0, 0, 0, 2, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171663); +tree.insert([0, 0, 0, 0, 0, 0, 1, 126], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171666); +tree.insert([0, 0, 0, 0, 0, 0, 2, 52], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171667); +tree.insert([0, 0, 0, 0, 0, 0, 2, 105], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171675); +tree.insert([0, 0, 0, 0, 0, 0, 1, 155], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171678); +tree.insert([0, 0, 0, 0, 0, 0, 0, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171679); +tree.insert([0, 0, 0, 0, 0, 0, 1, 118], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171681); +tree.insert([0, 0, 0, 0, 0, 0, 1, 69], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171686); +tree.insert([0, 0, 0, 0, 0, 0, 0, 81], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171687); +tree.insert([0, 0, 0, 0, 0, 0, 1, 254], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171692); +tree.insert([0, 0, 0, 0, 0, 0, 0, 35], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171695); +tree.insert([0, 0, 0, 0, 0, 0, 1, 102], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171699); +tree.insert([0, 0, 0, 0, 0, 0, 1, 247], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171700); +tree.insert([0, 0, 0, 0, 0, 0, 1, 174], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171703); +tree.insert([0, 0, 0, 0, 0, 0, 0, 215], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171705); +tree.insert([0, 0, 0, 0, 0, 0, 2, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171707); +tree.insert([0, 0, 0, 0, 0, 0, 0, 253], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171717); +tree.insert([0, 0, 0, 0, 0, 0, 1, 70], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171723); +tree.insert([0, 0, 0, 0, 0, 0, 2, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171724); +tree.insert([0, 0, 0, 0, 0, 0, 0, 156], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171725); +tree.insert([0, 0, 0, 0, 0, 0, 1, 52], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171728); +tree.insert([0, 0, 0, 0, 0, 0, 1, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171733); +tree.insert([0, 0, 0, 0, 0, 0, 0, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171735); +tree.insert([0, 0, 0, 0, 0, 0, 0, 72], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171740); +tree.insert([0, 0, 0, 0, 0, 0, 2, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171742); +tree.insert([0, 0, 0, 0, 0, 0, 1, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171746); +tree.insert([0, 0, 0, 0, 0, 0, 0, 14], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171747); +tree.insert([0, 0, 0, 0, 0, 0, 2, 166], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171751); +tree.insert([0, 0, 0, 0, 0, 0, 0, 162], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171752); +tree.insert([0, 0, 0, 0, 0, 0, 1, 26], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171754); +tree.insert([0, 0, 0, 0, 0, 0, 2, 138], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171763); +tree.insert([0, 0, 0, 0, 0, 0, 1, 248], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171764); +tree.insert([0, 0, 0, 0, 0, 0, 2, 4], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171765); +tree.insert([0, 0, 0, 0, 0, 0, 0, 7], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171767); +tree.insert([0, 0, 0, 0, 0, 0, 0, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171768); +tree.insert([0, 0, 0, 0, 0, 0, 0, 226], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171770); +tree.insert([0, 0, 0, 0, 0, 0, 1, 180], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171771); +tree.insert([0, 0, 0, 0, 0, 0, 0, 27], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171772); +tree.insert([0, 0, 0, 0, 0, 0, 2, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171777); +tree.insert([0, 0, 0, 0, 0, 0, 1, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171782); +tree.insert([0, 0, 0, 0, 0, 0, 2, 151], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171785); +tree.insert([0, 0, 0, 0, 0, 0, 1, 117], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171786); +tree.insert([0, 0, 0, 0, 0, 0, 0, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171787); +tree.insert([0, 0, 0, 0, 0, 0, 2, 186], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171788); +tree.insert([0, 0, 0, 0, 0, 0, 2, 134], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171792); +tree.insert([0, 0, 0, 0, 0, 0, 2, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171793); +tree.insert([0, 0, 0, 0, 0, 0, 1, 123], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171794); +tree.insert([0, 0, 0, 0, 0, 0, 1, 193], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171797); +tree.insert([0, 0, 0, 0, 0, 0, 1, 125], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171799); +tree.flush_active_memtable(171701)?; +tree.insert([0, 0, 0, 0, 0, 0, 1, 8], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171805); +tree.insert([0, 0, 0, 0, 0, 0, 2, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171806); +tree.insert([0, 0, 0, 0, 0, 0, 2, 27], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171809); +tree.insert([0, 0, 0, 0, 0, 0, 2, 102], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171810); +tree.insert([0, 0, 0, 0, 0, 0, 0, 228], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171811); +tree.insert([0, 0, 0, 0, 0, 0, 1, 67], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171812); +tree.insert([0, 0, 0, 0, 0, 0, 1, 189], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171813); +tree.insert([0, 0, 0, 0, 0, 0, 2, 84], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171814); +tree.insert([0, 0, 0, 0, 0, 0, 1, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171815); +tree.insert([0, 0, 0, 0, 0, 0, 0, 204], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171819); +tree.insert([0, 0, 0, 0, 0, 0, 1, 40], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171821); +tree.insert([0, 0, 0, 0, 0, 0, 1, 212], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171823); +tree.insert([0, 0, 0, 0, 0, 0, 2, 164], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171824); +tree.insert([0, 0, 0, 0, 0, 0, 1, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171825); +tree.insert([0, 0, 0, 0, 0, 0, 1, 104], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171831); +tree.insert([0, 0, 0, 0, 0, 0, 1, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171833); +tree.insert([0, 0, 0, 0, 0, 0, 0, 237], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171834); +tree.insert([0, 0, 0, 0, 0, 0, 2, 184], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171836); +tree.insert([0, 0, 0, 0, 0, 0, 2, 173], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171837); +tree.insert([0, 0, 0, 0, 0, 0, 2, 144], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171838); +tree.insert([0, 0, 0, 0, 0, 0, 2, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171848); +tree.insert([0, 0, 0, 0, 0, 0, 1, 187], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171850); +tree.insert([0, 0, 0, 0, 0, 0, 2, 168], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171856); +tree.insert([0, 0, 0, 0, 0, 0, 1, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171862); +tree.insert([0, 0, 0, 0, 0, 0, 0, 203], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171863); +tree.insert([0, 0, 0, 0, 0, 0, 0, 105], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171865); +tree.insert([0, 0, 0, 0, 0, 0, 2, 116], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171866); +tree.insert([0, 0, 0, 0, 0, 0, 1, 164], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171868); +tree.insert([0, 0, 0, 0, 0, 0, 1, 138], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171873); +tree.insert([0, 0, 0, 0, 0, 0, 0, 247], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171875); +tree.insert([0, 0, 0, 0, 0, 0, 1, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171877); +tree.insert([0, 0, 0, 0, 0, 0, 0, 190], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171882); +tree.insert([0, 0, 0, 0, 0, 0, 0, 90], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171885); +tree.insert([0, 0, 0, 0, 0, 0, 1, 11], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171890); +tree.insert([0, 0, 0, 0, 0, 0, 2, 66], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171891); +tree.insert([0, 0, 0, 0, 0, 0, 1, 132], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171892); +tree.insert([0, 0, 0, 0, 0, 0, 0, 36], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171895); +tree.insert([0, 0, 0, 0, 0, 0, 0, 122], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171898); +tree.insert([0, 0, 0, 0, 0, 0, 0, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171899); +tree.insert([0, 0, 0, 0, 0, 0, 0, 62], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171901); +tree.insert([0, 0, 0, 0, 0, 0, 2, 129], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171902); +tree.insert([0, 0, 0, 0, 0, 0, 0, 139], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171905); +tree.insert([0, 0, 0, 0, 0, 0, 1, 166], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171907); +tree.insert([0, 0, 0, 0, 0, 0, 0, 145], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171908); +tree.insert([0, 0, 0, 0, 0, 0, 1, 20], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171911); +tree.insert([0, 0, 0, 0, 0, 0, 2, 100], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171917); +tree.insert([0, 0, 0, 0, 0, 0, 0, 99], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171918); +tree.insert([0, 0, 0, 0, 0, 0, 1, 124], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171921); +tree.insert([0, 0, 0, 0, 0, 0, 2, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171922); +tree.insert([0, 0, 0, 0, 0, 0, 2, 149], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171925); +tree.insert([0, 0, 0, 0, 0, 0, 1, 246], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171926); +tree.insert([0, 0, 0, 0, 0, 0, 1, 95], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171928); +tree.insert([0, 0, 0, 0, 0, 0, 0, 76], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171931); +tree.insert([0, 0, 0, 0, 0, 0, 1, 9], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171936); +tree.insert([0, 0, 0, 0, 0, 0, 1, 152], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171937); +tree.insert([0, 0, 0, 0, 0, 0, 1, 158], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171947); +tree.insert([0, 0, 0, 0, 0, 0, 2, 51], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171948); +tree.insert([0, 0, 0, 0, 0, 0, 0, 51], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171950); +tree.insert([0, 0, 0, 0, 0, 0, 0, 144], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171958); +tree.insert([0, 0, 0, 0, 0, 0, 0, 244], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171959); +tree.insert([0, 0, 0, 0, 0, 0, 1, 109], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171963); +tree.insert([0, 0, 0, 0, 0, 0, 0, 59], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171965); +tree.insert([0, 0, 0, 0, 0, 0, 0, 76], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171975); +tree.insert([0, 0, 0, 0, 0, 0, 0, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171976); +tree.insert([0, 0, 0, 0, 0, 0, 1, 163], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171977); +tree.insert([0, 0, 0, 0, 0, 0, 1, 97], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171985); +tree.insert([0, 0, 0, 0, 0, 0, 1, 42], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171987); +tree.insert([0, 0, 0, 0, 0, 0, 1, 162], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171993); +tree.insert([0, 0, 0, 0, 0, 0, 1, 208], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 171997); +tree.flush_active_memtable(171901)?; +tree.compact(compaction.clone(), 171901)?; + + +tree.insert([0, 0, 0, 0, 0, 0, 3, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172007); +tree.insert([0, 0, 0, 0, 0, 0, 1, 37], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172008); +tree.insert([0, 0, 0, 0, 0, 0, 3, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172009); +tree.insert([0, 0, 0, 0, 0, 0, 3, 14], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172010); +tree.insert([0, 0, 0, 0, 0, 0, 2, 39], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172011); +tree.insert([0, 0, 0, 0, 0, 0, 2, 147], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172012); +tree.insert([0, 0, 0, 0, 0, 0, 2, 23], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172013); +tree.insert([0, 0, 0, 0, 0, 0, 0, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172014); +tree.insert([0, 0, 0, 0, 0, 0, 1, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172015); +tree.insert([0, 0, 0, 0, 0, 0, 2, 57], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172016); +tree.insert([0, 0, 0, 0, 0, 0, 0, 127], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172017); +tree.insert([0, 0, 0, 0, 0, 0, 1, 72], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172018); +tree.insert([0, 0, 0, 0, 0, 0, 3, 0], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172020); +tree.insert([0, 0, 0, 0, 0, 0, 1, 213], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172023); +tree.insert([0, 0, 0, 0, 0, 0, 0, 51], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172024); +tree.insert([0, 0, 0, 0, 0, 0, 2, 139], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172025); +tree.insert([0, 0, 0, 0, 0, 0, 1, 235], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172026); +tree.insert([0, 0, 0, 0, 0, 0, 2, 160], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172027); +tree.insert([0, 0, 0, 0, 0, 0, 2, 27], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172028); +tree.insert([0, 0, 0, 0, 0, 0, 1, 194], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172029); +tree.insert([0, 0, 0, 0, 0, 0, 1, 105], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172031); +tree.insert([0, 0, 0, 0, 0, 0, 1, 59], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172033); +tree.insert([0, 0, 0, 0, 0, 0, 2, 85], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172035); +tree.insert([0, 0, 0, 0, 0, 0, 1, 19], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172036); +tree.insert([0, 0, 0, 0, 0, 0, 2, 29], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172037); +tree.insert([0, 0, 0, 0, 0, 0, 2, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172038); +tree.insert([0, 0, 0, 0, 0, 0, 1, 18], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172039); +tree.insert([0, 0, 0, 0, 0, 0, 0, 0], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172043); +tree.insert([0, 0, 0, 0, 0, 0, 0, 144], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172044); +tree.insert([0, 0, 0, 0, 0, 0, 1, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172045); +tree.insert([0, 0, 0, 0, 0, 0, 1, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172046); +tree.insert([0, 0, 0, 0, 0, 0, 3, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172047); +tree.insert([0, 0, 0, 0, 0, 0, 2, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172048); +tree.insert([0, 0, 0, 0, 0, 0, 1, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172049); +tree.insert([0, 0, 0, 0, 0, 0, 2, 103], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172050); +tree.insert([0, 0, 0, 0, 0, 0, 1, 231], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172051); +tree.insert([0, 0, 0, 0, 0, 0, 2, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172052); +tree.insert([0, 0, 0, 0, 0, 0, 3, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172053); +tree.insert([0, 0, 0, 0, 0, 0, 1, 217], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172054); +tree.insert([0, 0, 0, 0, 0, 0, 1, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172056); +tree.insert([0, 0, 0, 0, 0, 0, 1, 184], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172057); +tree.insert([0, 0, 0, 0, 0, 0, 1, 212], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172058); +tree.insert([0, 0, 0, 0, 0, 0, 0, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172059); +tree.insert([0, 0, 0, 0, 0, 0, 1, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172061); +tree.insert([0, 0, 0, 0, 0, 0, 1, 173], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172062); +tree.insert([0, 0, 0, 0, 0, 0, 1, 237], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172064); +tree.insert([0, 0, 0, 0, 0, 0, 1, 25], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172065); +tree.insert([0, 0, 0, 0, 0, 0, 1, 125], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172066); +tree.insert([0, 0, 0, 0, 0, 0, 2, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172067); +tree.insert([0, 0, 0, 0, 0, 0, 1, 245], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172068); +tree.insert([0, 0, 0, 0, 0, 0, 1, 35], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172069); +tree.insert([0, 0, 0, 0, 0, 0, 3, 4], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172070); +tree.insert([0, 0, 0, 0, 0, 0, 3, 120], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172071); +tree.insert([0, 0, 0, 0, 0, 0, 3, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172072); +tree.insert([0, 0, 0, 0, 0, 0, 2, 74], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172073); +tree.insert([0, 0, 0, 0, 0, 0, 2, 15], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172074); +tree.insert([0, 0, 0, 0, 0, 0, 2, 109], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172075); +tree.insert([0, 0, 0, 0, 0, 0, 1, 39], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172077); +tree.insert([0, 0, 0, 0, 0, 0, 2, 221], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172078); +tree.insert([0, 0, 0, 0, 0, 0, 2, 250], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172081); +tree.insert([0, 0, 0, 0, 0, 0, 0, 180], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172082); +tree.insert([0, 0, 0, 0, 0, 0, 2, 5], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172083); +tree.insert([0, 0, 0, 0, 0, 0, 0, 223], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172085); +tree.insert([0, 0, 0, 0, 0, 0, 3, 109], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172087); +tree.insert([0, 0, 0, 0, 0, 0, 2, 175], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172088); +tree.insert([0, 0, 0, 0, 0, 0, 2, 228], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172089); +tree.insert([0, 0, 0, 0, 0, 0, 0, 126], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172090); +tree.insert([0, 0, 0, 0, 0, 0, 0, 56], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172091); +tree.insert([0, 0, 0, 0, 0, 0, 0, 157], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172092); +tree.insert([0, 0, 0, 0, 0, 0, 0, 99], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172093); +tree.insert([0, 0, 0, 0, 0, 0, 0, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172094); +tree.insert([0, 0, 0, 0, 0, 0, 0, 76], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172095); +tree.insert([0, 0, 0, 0, 0, 0, 2, 253], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172096); +tree.insert([0, 0, 0, 0, 0, 0, 0, 210], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172097); +tree.insert([0, 0, 0, 0, 0, 0, 3, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172098); +tree.insert([0, 0, 0, 0, 0, 0, 2, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172101); +tree.insert([0, 0, 0, 0, 0, 0, 2, 118], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172102); +tree.insert([0, 0, 0, 0, 0, 0, 0, 85], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172104); +tree.insert([0, 0, 0, 0, 0, 0, 2, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172105); +tree.insert([0, 0, 0, 0, 0, 0, 2, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172108); +tree.insert([0, 0, 0, 0, 0, 0, 1, 249], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172110); +tree.insert([0, 0, 0, 0, 0, 0, 1, 243], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172111); +tree.insert([0, 0, 0, 0, 0, 0, 1, 48], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172112); +tree.insert([0, 0, 0, 0, 0, 0, 1, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172113); +tree.insert([0, 0, 0, 0, 0, 0, 2, 121], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172114); +tree.insert([0, 0, 0, 0, 0, 0, 1, 2], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172116); +tree.insert([0, 0, 0, 0, 0, 0, 0, 8], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172117); +tree.insert([0, 0, 0, 0, 0, 0, 1, 128], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172118); +tree.insert([0, 0, 0, 0, 0, 0, 1, 104], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172119); +tree.insert([0, 0, 0, 0, 0, 0, 0, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172122); +tree.insert([0, 0, 0, 0, 0, 0, 0, 122], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172123); +tree.insert([0, 0, 0, 0, 0, 0, 0, 123], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172124); +tree.insert([0, 0, 0, 0, 0, 0, 2, 140], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172125); +tree.insert([0, 0, 0, 0, 0, 0, 2, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172126); +tree.insert([0, 0, 0, 0, 0, 0, 0, 235], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172127); +tree.insert([0, 0, 0, 0, 0, 0, 1, 126], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172128); +tree.insert([0, 0, 0, 0, 0, 0, 1, 149], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172149); +tree.insert([0, 0, 0, 0, 0, 0, 0, 9], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172151); +tree.insert([0, 0, 0, 0, 0, 0, 1, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172152); +tree.insert([0, 0, 0, 0, 0, 0, 0, 64], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172153); +tree.insert([0, 0, 0, 0, 0, 0, 0, 161], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172157); +tree.insert([0, 0, 0, 0, 0, 0, 1, 200], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172158); +tree.insert([0, 0, 0, 0, 0, 0, 1, 234], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172163); +tree.insert([0, 0, 0, 0, 0, 0, 1, 196], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172164); +tree.insert([0, 0, 0, 0, 0, 0, 1, 153], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172167); +tree.flush_active_memtable(172101)?; +tree.compact(compaction.clone(), 172101)?; +tree.insert([0, 0, 0, 0, 0, 0, 0, 219], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172222); +tree.insert([0, 0, 0, 0, 0, 0, 0, 124], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172225); +tree.insert([0, 0, 0, 0, 0, 0, 2, 192], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172226); +tree.insert([0, 0, 0, 0, 0, 0, 0, 108], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172227); +tree.insert([0, 0, 0, 0, 0, 0, 0, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172229); +tree.insert([0, 0, 0, 0, 0, 0, 0, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172231); +tree.insert([0, 0, 0, 0, 0, 0, 1, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172233); +tree.insert([0, 0, 0, 0, 0, 0, 2, 100], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172235); +tree.insert([0, 0, 0, 0, 0, 0, 3, 40], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172237); +tree.insert([0, 0, 0, 0, 0, 0, 3, 39], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172238); +tree.insert([0, 0, 0, 0, 0, 0, 2, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172239); +tree.insert([0, 0, 0, 0, 0, 0, 1, 172], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172240); +tree.insert([0, 0, 0, 0, 0, 0, 0, 143], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172242); +tree.insert([0, 0, 0, 0, 0, 0, 1, 101], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172245); +tree.insert([0, 0, 0, 0, 0, 0, 0, 138], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172247); +tree.insert([0, 0, 0, 0, 0, 0, 0, 88], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172251); +tree.insert([0, 0, 0, 0, 0, 0, 0, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172252); +tree.insert([0, 0, 0, 0, 0, 0, 2, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172253); +tree.insert([0, 0, 0, 0, 0, 0, 1, 233], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172257); +tree.insert([0, 0, 0, 0, 0, 0, 0, 229], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172259); +tree.insert([0, 0, 0, 0, 0, 0, 1, 32], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172260); +tree.insert([0, 0, 0, 0, 0, 0, 1, 211], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172261); +tree.insert([0, 0, 0, 0, 0, 0, 2, 69], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172262); +tree.insert([0, 0, 0, 0, 0, 0, 2, 86], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172263); +tree.insert([0, 0, 0, 0, 0, 0, 2, 56], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172264); +tree.insert([0, 0, 0, 0, 0, 0, 2, 190], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172265); +tree.insert([0, 0, 0, 0, 0, 0, 2, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172270); +tree.insert([0, 0, 0, 0, 0, 0, 2, 21], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172271); +tree.insert([0, 0, 0, 0, 0, 0, 1, 118], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172273); +tree.insert([0, 0, 0, 0, 0, 0, 1, 193], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172274); +tree.insert([0, 0, 0, 0, 0, 0, 0, 115], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172275); +tree.insert([0, 0, 0, 0, 0, 0, 2, 200], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172276); +tree.insert([0, 0, 0, 0, 0, 0, 0, 225], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172278); +tree.insert([0, 0, 0, 0, 0, 0, 1, 7], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172279); +tree.insert([0, 0, 0, 0, 0, 0, 0, 129], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172280); +tree.insert([0, 0, 0, 0, 0, 0, 0, 226], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172281); +tree.insert([0, 0, 0, 0, 0, 0, 2, 9], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172286); +tree.insert([0, 0, 0, 0, 0, 0, 0, 50], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172289); +tree.insert([0, 0, 0, 0, 0, 0, 2, 43], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172290); +tree.insert([0, 0, 0, 0, 0, 0, 0, 120], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172291); +tree.insert([0, 0, 0, 0, 0, 0, 1, 239], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172292); +tree.insert([0, 0, 0, 0, 0, 0, 0, 61], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172293); +tree.insert([0, 0, 0, 0, 0, 0, 0, 246], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172294); +tree.insert([0, 0, 0, 0, 0, 0, 0, 7], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172296); +tree.insert([0, 0, 0, 0, 0, 0, 0, 68], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172297); +tree.insert([0, 0, 0, 0, 0, 0, 3, 18], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172298); +tree.insert([0, 0, 0, 0, 0, 0, 0, 131], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172299); +tree.insert([0, 0, 0, 0, 0, 0, 2, 124], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172301); +tree.insert([0, 0, 0, 0, 0, 0, 0, 142], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172304); +tree.insert([0, 0, 0, 0, 0, 0, 2, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172305); +tree.insert([0, 0, 0, 0, 0, 0, 1, 42], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172307); +tree.insert([0, 0, 0, 0, 0, 0, 0, 128], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172308); +tree.insert([0, 0, 0, 0, 0, 0, 3, 36], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172310); +tree.insert([0, 0, 0, 0, 0, 0, 0, 184], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172311); +tree.insert([0, 0, 0, 0, 0, 0, 1, 182], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172313); +tree.insert([0, 0, 0, 0, 0, 0, 1, 5], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172315); +tree.insert([0, 0, 0, 0, 0, 0, 2, 198], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172317); +tree.insert([0, 0, 0, 0, 0, 0, 0, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172318); +tree.insert([0, 0, 0, 0, 0, 0, 0, 39], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172321); +tree.insert([0, 0, 0, 0, 0, 0, 2, 36], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172323); +tree.insert([0, 0, 0, 0, 0, 0, 0, 192], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172324); +tree.insert([0, 0, 0, 0, 0, 0, 1, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172327); +tree.insert([0, 0, 0, 0, 0, 0, 1, 181], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172328); +tree.insert([0, 0, 0, 0, 0, 0, 0, 116], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172331); +tree.insert([0, 0, 0, 0, 0, 0, 0, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172337); +tree.insert([0, 0, 0, 0, 0, 0, 1, 30], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172338); +tree.insert([0, 0, 0, 0, 0, 0, 0, 243], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172340); +tree.insert([0, 0, 0, 0, 0, 0, 0, 90], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172342); +tree.insert([0, 0, 0, 0, 0, 0, 0, 178], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172343); +tree.insert([0, 0, 0, 0, 0, 0, 1, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172350); +tree.insert([0, 0, 0, 0, 0, 0, 0, 240], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172352); +tree.insert([0, 0, 0, 0, 0, 0, 2, 8], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172354); +tree.insert([0, 0, 0, 0, 0, 0, 2, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172355); +tree.insert([0, 0, 0, 0, 0, 0, 2, 87], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172357); +tree.insert([0, 0, 0, 0, 0, 0, 1, 127], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172358); +tree.insert([0, 0, 0, 0, 0, 0, 0, 195], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172359); +tree.insert([0, 0, 0, 0, 0, 0, 2, 115], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172360); +tree.insert([0, 0, 0, 0, 0, 0, 2, 126], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172362); +tree.insert([0, 0, 0, 0, 0, 0, 2, 105], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172365); +tree.insert([0, 0, 0, 0, 0, 0, 2, 191], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172366); +tree.insert([0, 0, 0, 0, 0, 0, 1, 209], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172367); +tree.insert([0, 0, 0, 0, 0, 0, 0, 156], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172368); +tree.insert([0, 0, 0, 0, 0, 0, 2, 208], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172370); +tree.insert([0, 0, 0, 0, 0, 0, 2, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172371); +tree.insert([0, 0, 0, 0, 0, 0, 1, 87], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172372); +tree.insert([0, 0, 0, 0, 0, 0, 1, 44], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172373); +tree.insert([0, 0, 0, 0, 0, 0, 0, 232], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172374); +tree.insert([0, 0, 0, 0, 0, 0, 2, 47], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172376); +tree.insert([0, 0, 0, 0, 0, 0, 1, 178], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172378); +tree.insert([0, 0, 0, 0, 0, 0, 2, 150], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172380); +tree.insert([0, 0, 0, 0, 0, 0, 2, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172381); +tree.insert([0, 0, 0, 0, 0, 0, 1, 53], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172383); +tree.insert([0, 0, 0, 0, 0, 0, 1, 81], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172387); +tree.insert([0, 0, 0, 0, 0, 0, 1, 227], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172390); +tree.insert([0, 0, 0, 0, 0, 0, 0, 38], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172392); +tree.insert([0, 0, 0, 0, 0, 0, 2, 162], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172393); +tree.insert([0, 0, 0, 0, 0, 0, 1, 65], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172394); +tree.insert([0, 0, 0, 0, 0, 0, 1, 255], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172397); +tree.insert([0, 0, 0, 0, 0, 0, 0, 95], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172398); +tree.insert([0, 0, 0, 0, 0, 0, 3, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172399); +tree.insert([0, 0, 0, 0, 0, 0, 0, 66], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172400); +tree.flush_active_memtable(172301)?; +tree.insert([0, 0, 0, 0, 0, 0, 1, 27], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172402); +tree.insert([0, 0, 0, 0, 0, 0, 0, 187], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172404); +tree.insert([0, 0, 0, 0, 0, 0, 2, 211], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172408); +tree.insert([0, 0, 0, 0, 0, 0, 1, 216], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172409); +tree.insert([0, 0, 0, 0, 0, 0, 0, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172414); +tree.insert([0, 0, 0, 0, 0, 0, 0, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172416); +tree.insert([0, 0, 0, 0, 0, 0, 3, 7], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172417); +tree.insert([0, 0, 0, 0, 0, 0, 1, 176], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172418); +tree.insert([0, 0, 0, 0, 0, 0, 0, 215], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172420); +tree.insert([0, 0, 0, 0, 0, 0, 1, 223], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172423); +tree.insert([0, 0, 0, 0, 0, 0, 0, 221], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172424); +tree.insert([0, 0, 0, 0, 0, 0, 0, 54], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172425); +tree.insert([0, 0, 0, 0, 0, 0, 0, 166], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172426); +tree.insert([0, 0, 0, 0, 0, 0, 1, 167], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172427); +tree.insert([0, 0, 0, 0, 0, 0, 2, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172428); +tree.insert([0, 0, 0, 0, 0, 0, 2, 217], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172429); +tree.insert([0, 0, 0, 0, 0, 0, 1, 56], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172432); +tree.insert([0, 0, 0, 0, 0, 0, 0, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172433); +tree.insert([0, 0, 0, 0, 0, 0, 2, 89], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172434); +tree.insert([0, 0, 0, 0, 0, 0, 1, 96], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172435); +tree.insert([0, 0, 0, 0, 0, 0, 1, 58], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172436); +tree.insert([0, 0, 0, 0, 0, 0, 2, 88], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172438); +tree.insert([0, 0, 0, 0, 0, 0, 0, 188], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172439); +tree.insert([0, 0, 0, 0, 0, 0, 1, 99], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172440); +tree.insert([0, 0, 0, 0, 0, 0, 2, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172441); +tree.insert([0, 0, 0, 0, 0, 0, 1, 83], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172442); +tree.insert([0, 0, 0, 0, 0, 0, 2, 49], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172443); +tree.insert([0, 0, 0, 0, 0, 0, 1, 13], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172444); +tree.insert([0, 0, 0, 0, 0, 0, 0, 211], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172445); +tree.insert([0, 0, 0, 0, 0, 0, 0, 227], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172449); +tree.insert([0, 0, 0, 0, 0, 0, 3, 22], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172450); +tree.insert([0, 0, 0, 0, 0, 0, 0, 230], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172451); +tree.insert([0, 0, 0, 0, 0, 0, 1, 112], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172455); +tree.insert([0, 0, 0, 0, 0, 0, 0, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172457); +tree.insert([0, 0, 0, 0, 0, 0, 2, 224], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172458); +tree.insert([0, 0, 0, 0, 0, 0, 1, 124], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172461); +tree.insert([0, 0, 0, 0, 0, 0, 1, 111], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172462); +tree.insert([0, 0, 0, 0, 0, 0, 0, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172463); +tree.insert([0, 0, 0, 0, 0, 0, 0, 253], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172465); +tree.insert([0, 0, 0, 0, 0, 0, 1, 8], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172466); +tree.insert([0, 0, 0, 0, 0, 0, 1, 85], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172470); +tree.insert([0, 0, 0, 0, 0, 0, 1, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172471); +tree.insert([0, 0, 0, 0, 0, 0, 1, 250], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172472); +tree.insert([0, 0, 0, 0, 0, 0, 1, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172479); +tree.insert([0, 0, 0, 0, 0, 0, 1, 138], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172486); +tree.insert([0, 0, 0, 0, 0, 0, 0, 46], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172488); +tree.insert([0, 0, 0, 0, 0, 0, 2, 41], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172489); +tree.insert([0, 0, 0, 0, 0, 0, 1, 224], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172490); +tree.insert([0, 0, 0, 0, 0, 0, 1, 228], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172491); +tree.insert([0, 0, 0, 0, 0, 0, 2, 35], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172493); +tree.insert([0, 0, 0, 0, 0, 0, 0, 75], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172494); +tree.insert([0, 0, 0, 0, 0, 0, 1, 183], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172495); +tree.insert([0, 0, 0, 0, 0, 0, 1, 20], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172496); +tree.insert([0, 0, 0, 0, 0, 0, 2, 243], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172497); +tree.insert([0, 0, 0, 0, 0, 0, 3, 46], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172498); +tree.insert([0, 0, 0, 0, 0, 0, 3, 55], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172508); +tree.insert([0, 0, 0, 0, 0, 0, 1, 195], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172510); +tree.insert([0, 0, 0, 0, 0, 0, 2, 222], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172511); +tree.insert([0, 0, 0, 0, 0, 0, 1, 97], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172512); +tree.insert([0, 0, 0, 0, 0, 0, 0, 134], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172513); +tree.insert([0, 0, 0, 0, 0, 0, 0, 206], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172514); +tree.insert([0, 0, 0, 0, 0, 0, 1, 236], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172516); +tree.insert([0, 0, 0, 0, 0, 0, 3, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172519); +tree.insert([0, 0, 0, 0, 0, 0, 2, 7], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172523); +tree.insert([0, 0, 0, 0, 0, 0, 2, 13], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172525); +tree.insert([0, 0, 0, 0, 0, 0, 1, 201], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172527); +tree.insert([0, 0, 0, 0, 0, 0, 1, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172528); +tree.insert([0, 0, 0, 0, 0, 0, 1, 156], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172529); +tree.insert([0, 0, 0, 0, 0, 0, 1, 246], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172535); +tree.insert([0, 0, 0, 0, 0, 0, 0, 78], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172536); +tree.insert([0, 0, 0, 0, 0, 0, 1, 117], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172537); +tree.insert([0, 0, 0, 0, 0, 0, 1, 62], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172538); +tree.insert([0, 0, 0, 0, 0, 0, 0, 94], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172541); +tree.insert([0, 0, 0, 0, 0, 0, 2, 71], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172542); +tree.insert([0, 0, 0, 0, 0, 0, 1, 145], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172547); +tree.insert([0, 0, 0, 0, 0, 0, 2, 178], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172548); +tree.insert([0, 0, 0, 0, 0, 0, 0, 252], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172549); +tree.insert([0, 0, 0, 0, 0, 0, 2, 154], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172551); +tree.insert([0, 0, 0, 0, 0, 0, 1, 140], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172554); +tree.insert([0, 0, 0, 0, 0, 0, 2, 98], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172555); +tree.insert([0, 0, 0, 0, 0, 0, 0, 24], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172556); +tree.insert([0, 0, 0, 0, 0, 0, 2, 159], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172558); +tree.insert([0, 0, 0, 0, 0, 0, 2, 220], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172559); +tree.insert([0, 0, 0, 0, 0, 0, 3, 43], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172560); +tree.insert([0, 0, 0, 0, 0, 0, 0, 249], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172562); +tree.insert([0, 0, 0, 0, 0, 0, 0, 11], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172563); +tree.insert([0, 0, 0, 0, 0, 0, 3, 19], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172566); +tree.insert([0, 0, 0, 0, 0, 0, 1, 36], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172567); +tree.insert([0, 0, 0, 0, 0, 0, 0, 19], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172570); +tree.insert([0, 0, 0, 0, 0, 0, 2, 171], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172574); +tree.insert([0, 0, 0, 0, 0, 0, 0, 222], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172579); +tree.insert([0, 0, 0, 0, 0, 0, 0, 201], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172582); +tree.insert([0, 0, 0, 0, 0, 0, 1, 119], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172584); +tree.insert([0, 0, 0, 0, 0, 0, 2, 113], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172586); +tree.insert([0, 0, 0, 0, 0, 0, 1, 130], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172587); +tree.insert([0, 0, 0, 0, 0, 0, 1, 17], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172591); +tree.insert([0, 0, 0, 0, 0, 0, 1, 91], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172595); +tree.insert([0, 0, 0, 0, 0, 0, 0, 33], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172599); +tree.flush_active_memtable(172501)?; +tree.insert([0, 0, 0, 0, 0, 0, 0, 114], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172602); +tree.insert([0, 0, 0, 0, 0, 0, 1, 113], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172605); +tree.insert([0, 0, 0, 0, 0, 0, 0, 100], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172609); +tree.insert([0, 0, 0, 0, 0, 0, 0, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172610); +tree.insert([0, 0, 0, 0, 0, 0, 2, 117], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172614); +tree.insert([0, 0, 0, 0, 0, 0, 2, 52], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172615); +tree.insert([0, 0, 0, 0, 0, 0, 3, 35], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172616); +tree.insert([0, 0, 0, 0, 0, 0, 0, 231], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172619); +tree.insert([0, 0, 0, 0, 0, 0, 0, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172625); +tree.insert([0, 0, 0, 0, 0, 0, 1, 147], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172626); +tree.insert([0, 0, 0, 0, 0, 0, 3, 3], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172627); +tree.insert([0, 0, 0, 0, 0, 0, 1, 247], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172628); +tree.insert([0, 0, 0, 0, 0, 0, 1, 190], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172631); +tree.insert([0, 0, 0, 0, 0, 0, 1, 164], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172632); +tree.insert([0, 0, 0, 0, 0, 0, 1, 137], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172633); +tree.insert([0, 0, 0, 0, 0, 0, 1, 168], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172634); +tree.insert([0, 0, 0, 0, 0, 0, 3, 12], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172637); +tree.insert([0, 0, 0, 0, 0, 0, 2, 183], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172638); +tree.insert([0, 0, 0, 0, 0, 0, 0, 113], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172645); +tree.insert([0, 0, 0, 0, 0, 0, 0, 233], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172649); +tree.insert([0, 0, 0, 0, 0, 0, 2, 67], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172650); +tree.insert([0, 0, 0, 0, 0, 0, 1, 203], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172655); +tree.insert([0, 0, 0, 0, 0, 0, 2, 80], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172656); +tree.insert([0, 0, 0, 0, 0, 0, 0, 179], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172658); +tree.insert([0, 0, 0, 0, 0, 0, 1, 45], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172660); +tree.insert([0, 0, 0, 0, 0, 0, 0, 2], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172661); +tree.insert([0, 0, 0, 0, 0, 0, 1, 93], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172663); +tree.insert([0, 0, 0, 0, 0, 0, 1, 28], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172665); +tree.insert([0, 0, 0, 0, 0, 0, 0, 164], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172672); +tree.insert([0, 0, 0, 0, 0, 0, 0, 60], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172675); +tree.insert([0, 0, 0, 0, 0, 0, 0, 170], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172679); +tree.insert([0, 0, 0, 0, 0, 0, 0, 92], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172684); +tree.insert([0, 0, 0, 0, 0, 0, 1, 197], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172688); +tree.insert([0, 0, 0, 0, 0, 0, 0, 169], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172693); +tree.insert([0, 0, 0, 0, 0, 0, 3, 23], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172700); +tree.insert([0, 0, 0, 0, 0, 0, 0, 153], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172702); +tree.insert([0, 0, 0, 0, 0, 0, 2, 4], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172704); +tree.insert([0, 0, 0, 0, 0, 0, 0, 84], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172705); +tree.insert([0, 0, 0, 0, 0, 0, 0, 250], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172706); +tree.insert([0, 0, 0, 0, 0, 0, 1, 218], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172708); +tree.insert([0, 0, 0, 0, 0, 0, 0, 132], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172728); +tree.insert([0, 0, 0, 0, 0, 0, 1, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172733); +tree.insert([0, 0, 0, 0, 0, 0, 0, 63], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172735); +tree.insert([0, 0, 0, 0, 0, 0, 0, 77], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172737); +tree.insert([0, 0, 0, 0, 0, 0, 0, 35], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172740); +tree.insert([0, 0, 0, 0, 0, 0, 0, 103], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172742); +tree.insert([0, 0, 0, 0, 0, 0, 0, 16], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172749); +tree.insert([0, 0, 0, 0, 0, 0, 1, 38], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172750); +tree.insert([0, 0, 0, 0, 0, 0, 0, 155], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172755); +tree.insert([0, 0, 0, 0, 0, 0, 0, 255], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172756); +tree.insert([0, 0, 0, 0, 0, 0, 1, 46], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172761); +tree.insert([0, 0, 0, 0, 0, 0, 0, 10], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172765); +tree.insert([0, 0, 0, 0, 0, 0, 0, 82], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172766); +tree.insert([0, 0, 0, 0, 0, 0, 0, 177], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172767); +tree.insert([0, 0, 0, 0, 0, 0, 0, 199], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172774); +tree.insert([0, 0, 0, 0, 0, 0, 0, 106], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172786); +tree.insert([0, 0, 0, 0, 0, 0, 1, 34], [104, 101, 108, 108, 111, 104, 101, 108, 108, 111], 172788); +tree.flush_active_memtable(172701)?; +tree.compact(compaction.clone(), 172701)?; + +tree.compact(compaction.clone(), 172901)?; + +tree.compact(compaction.clone(), 173101)?; + + tree.drop_range::<&[u8], _>(..)?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/multi_trees.rs b/crates/lsm-tree/tests/multi_trees.rs new file mode 100644 index 000000000..5e516b277 --- /dev/null +++ b/crates/lsm-tree/tests/multi_trees.rs @@ -0,0 +1,69 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +#[ignore] +fn tree_multi_table_ids() -> lsm_tree::Result<()> { + let folder0 = get_tmp_folder(); + let folder1 = get_tmp_folder(); + + let tree0 = Config::new( + &folder0, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(tree0.id(), 0); + + assert_eq!(0, tree0.next_table_id()); + + tree0.insert("a", "a", 0); + tree0.flush_active_memtable(0)?; + + assert_eq!(2, tree0.next_table_id()); + + assert_eq!( + 1, + tree0 + .current_version() + .level(0) + .expect("level should exist") + .first() + .expect("run should exist") + .first() + .expect("table should exist") + .metadata + .id + ); + + let tree1 = Config::new( + &folder1, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(tree1.id(), 1); + + assert_eq!(0, tree1.next_table_id()); + + tree1.insert("a", "a", 0); + tree1.flush_active_memtable(0)?; + + assert_eq!(2, tree1.next_table_id()); + + assert_eq!( + 1, + tree1 + .current_version() + .level(0) + .expect("level should exist") + .first() + .expect("run should exist") + .first() + .expect("table should exist") + .metadata + .id + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/mvcc_slab.rs b/crates/lsm-tree/tests/mvcc_slab.rs new file mode 100644 index 000000000..223ac5659 --- /dev/null +++ b/crates/lsm-tree/tests/mvcc_slab.rs @@ -0,0 +1,77 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn table_reader_mvcc_slab() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 10_000; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for _ in 0..ITEM_COUNT { + tree.insert("a", "", seqno.next()); + } + tree.insert("b", "", 0); + + tree.flush_active_memtable(0)?; + + let version = tree.current_version(); + + let table = version + .level(0) + .expect("level should exist") + .first() + .expect("run should exist") + .first() + .expect("table should exist"); + + let reader = table.iter(); + assert_eq!(reader.count(), ITEM_COUNT + 1); + + Ok(()) +} + +#[test] +fn table_reader_mvcc_slab_blob() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 1_000; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .with_kv_separation(Some(Default::default())) + .open()?; + + for _ in 0..ITEM_COUNT { + tree.insert("a", "neptune".repeat(10_000), seqno.next()); + } + tree.insert("b", "", 0); + + tree.flush_active_memtable(0)?; + + let version = tree.current_version(); + + let table = version + .level(0) + .expect("level should exist") + .first() + .expect("run should exist") + .first() + .expect("table should exist"); + + let reader = table.iter(); + assert_eq!(reader.count(), ITEM_COUNT + 1); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/recover_cleanup_orphans.rs b/crates/lsm-tree/tests/recover_cleanup_orphans.rs new file mode 100644 index 000000000..b9f1d2f71 --- /dev/null +++ b/crates/lsm-tree/tests/recover_cleanup_orphans.rs @@ -0,0 +1,90 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_recovery_cleanup_orphans() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + assert!(folder.path().join("tables").join("0").try_exists()?); + + tree.major_compact(u64::MAX, 0)?; + + assert!(folder.path().join("tables").join("1").try_exists()?); + } + + std::fs::File::create(folder.path().join("tables").join("0"))?; + + { + let _tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert!(!folder.path().join("tables").join("0").try_exists()?); + assert!(folder.path().join("tables").join("1").try_exists()?); + } + + Ok(()) +} + +#[test] +fn tree_recovery_cleanup_orphans_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .age_cutoff(1.0) + .staleness_threshold(0.01), + )) + .open()?; + + tree.insert("a", "a".repeat(10_000), 0); + tree.insert("a", "a".repeat(10_000), 1); + tree.flush_active_memtable(0)?; + + assert!(folder.path().join("blobs").join("0").try_exists()?); + + tree.major_compact(u64::MAX, 5)?; + + assert!(folder.path().join("blobs").join("0").try_exists()?); + + tree.major_compact(u64::MAX, 10)?; + + assert!(folder.path().join("blobs").join("1").try_exists()?); + } + + std::fs::File::create(folder.path().join("blobs").join("0"))?; + + { + let _tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert!(!folder.path().join("blobs").join("0").try_exists()?); + assert!(folder.path().join("blobs").join("1").try_exists()?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/recover_from_different_folder.rs b/crates/lsm-tree/tests/recover_from_different_folder.rs new file mode 100644 index 000000000..c1fbe195e --- /dev/null +++ b/crates/lsm-tree/tests/recover_from_different_folder.rs @@ -0,0 +1,74 @@ +use lsm_tree::{AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn recover_from_different_folder() -> lsm_tree::Result<()> { + if std::path::Path::new(".test").try_exists()? { + std::fs::remove_dir_all(".test")?; + } + + let folder = ".test/asd"; + + { + let tree = Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("abc", "def", 0); + tree.insert("wqewe", "def", 0); + tree.insert("ewewq", "def", 0); + tree.insert("asddas", "def", 0); + tree.insert("ycxycx", "def", 0); + tree.insert("asdsda", "def", 0); + tree.insert("wewqe", "def", 0); + + tree.flush_active_memtable(0)?; + } + + { + let _tree = Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + } + + let absolute_folder = std::path::Path::new(folder).canonicalize()?; + + std::fs::create_dir_all(".test/def")?; + std::env::set_current_dir(".test/def")?; + + { + let tree = Config::new( + &absolute_folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("abc", "def", 0); + tree.insert("wqewe", "def", 0); + tree.insert("ewewq", "def", 0); + tree.insert("asddas", "def", 0); + tree.insert("ycxycx", "def", 0); + tree.insert("asdsda", "def", 0); + tree.insert("wewqe", "def", 0); + + tree.flush_active_memtable(0)?; + } + + for _ in 0..10 { + let _tree = Config::new( + &absolute_folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/recovery_mac_ds_store.rs b/crates/lsm-tree/tests/recovery_mac_ds_store.rs new file mode 100644 index 000000000..b767b60dc --- /dev/null +++ b/crates/lsm-tree/tests/recovery_mac_ds_store.rs @@ -0,0 +1,36 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn recovery_mac_ds_store() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + } + + let ds_store = folder.path().join("tables").join(".DS_Store"); + std::fs::File::create(&ds_store)?; + assert!(ds_store.try_exists()?); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(1, tree.table_count()); + } + assert!(ds_store.try_exists()?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/recovery_mac_underscore.rs b/crates/lsm-tree/tests/recovery_mac_underscore.rs new file mode 100644 index 000000000..b32315c1c --- /dev/null +++ b/crates/lsm-tree/tests/recovery_mac_underscore.rs @@ -0,0 +1,36 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn recovery_mac_underscore_file() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + } + + let ds_store = folder.path().join("tables").join("._0"); + std::fs::File::create(&ds_store)?; + assert!(ds_store.try_exists()?); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(1, tree.table_count()); + } + assert!(ds_store.try_exists()?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/regression_286.rs b/crates/lsm-tree/tests/regression_286.rs new file mode 100644 index 000000000..f692262b0 --- /dev/null +++ b/crates/lsm-tree/tests/regression_286.rs @@ -0,0 +1,32 @@ +use lsm_tree::AbstractTree; + +#[test_log::test] +fn regression_286() -> lsm_tree::Result<()> { + let folder = lsm_tree::get_tmp_folder(); + let seqno = lsm_tree::SequenceNumberCounter::default(); + + { + let tree = lsm_tree::Config::new( + &folder, + seqno.clone(), + lsm_tree::SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(lsm_tree::KvSeparationOptions::default())) + .open()?; + + tree.insert("foo", b"1", 0); + tree.clear()?; + } + + { + let _tree = lsm_tree::Config::new( + &folder, + seqno.clone(), + lsm_tree::SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(lsm_tree::KvSeparationOptions::default())) + .open()?; + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/snapshot_compact.rs b/crates/lsm-tree/tests/snapshot_compact.rs new file mode 100644 index 000000000..60fbef86b --- /dev/null +++ b/crates/lsm-tree/tests/snapshot_compact.rs @@ -0,0 +1,61 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn snapshot_after_compaction_simple() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a", "a", seqno.next()); + + let snapshot_seqno = seqno.get(); + assert_eq!(b"a", &*tree.get("a", u64::MAX)?.unwrap()); + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + tree.insert("a", "b", seqno.next()); + tree.flush_active_memtable(0)?; + assert_eq!(b"b", &*tree.get("a", u64::MAX)?.unwrap()); + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + tree.major_compact(u64::MAX, 0)?; + assert_eq!(b"b", &*tree.get("a", u64::MAX)?.unwrap()); + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + + Ok(()) +} + +#[test] +fn snapshot_after_compaction_iters() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 100; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + let snapshot_seqno = seqno.get(); + assert_eq!(ITEM_COUNT, tree.len(snapshot_seqno, None)?); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + tree.flush_active_memtable(0)?; + tree.major_compact(u64::MAX, 0)?; + + assert_eq!(tree.len(seqno.get(), None)?, ITEM_COUNT); + + assert_eq!(ITEM_COUNT, tree.len(snapshot_seqno, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/snapshot_len.rs b/crates/lsm-tree/tests/snapshot_len.rs new file mode 100644 index 000000000..636b8520e --- /dev/null +++ b/crates/lsm-tree/tests/snapshot_len.rs @@ -0,0 +1,41 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +const ITEM_COUNT: usize = 100; + +#[test] +fn snapshot_basic() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + + let instant = seqno.get(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, tree.len(instant, None)?); + + for x in (ITEM_COUNT as u64)..((ITEM_COUNT * 2) as u64) { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!(ITEM_COUNT, tree.len(instant, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/snapshot_point_read.rs b/crates/lsm-tree/tests/snapshot_point_read.rs new file mode 100644 index 000000000..0a4f5aa7d --- /dev/null +++ b/crates/lsm-tree/tests/snapshot_point_read.rs @@ -0,0 +1,189 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +#[test] +#[ignore] +fn snapshot_404() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + let batch_seqno = seqno.next(); + tree.insert("a", "a", batch_seqno); + tree.insert("a2", "a2", batch_seqno); + tree.insert("c", "c", batch_seqno); + + tree.flush_active_memtable(0)?; + + let snapshot_seqno = seqno.get(); + assert_eq!(b"a", &*tree.get("a", snapshot_seqno)?.unwrap()); + assert_eq!(b"a2", &*tree.get("a2", snapshot_seqno)?.unwrap()); + assert!(tree.get("b", snapshot_seqno)?.is_none()); + assert_eq!(b"c", &*tree.get("c", snapshot_seqno)?.unwrap()); + + assert!(tree.get("a", 0)?.is_none()); + assert!(tree.get("a2", 0)?.is_none()); + assert!(tree.get("b", 0)?.is_none()); + assert!(tree.get("c", 0)?.is_none()); + + assert_eq!(b"a2", &*tree.get("a2", SeqNo::MAX)?.unwrap()); + assert!(tree.get("b", SeqNo::MAX)?.is_none()); + assert_eq!(b"c", &*tree.get("c", SeqNo::MAX)?.unwrap()); + + Ok(()) +} + +#[test] +fn snapshot_lots_of_versions() -> lsm_tree::Result<()> { + let version_count = 600; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + let key = "abc"; + + for seqno in 0u64..version_count { + tree.insert(key, format!("abc{version_count}").as_bytes(), seqno); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + + for seqno in 1..version_count { + let item = tree + .get_internal_entry(key.as_bytes(), seqno)? + .expect("should exist"); + + assert_eq!(format!("abc{}", version_count).as_bytes(), &*item.value); + + let item = tree.get(key, SeqNo::MAX)?.expect("should exist"); + assert_eq!(format!("abc{}", version_count).as_bytes(), &*item); + } + + Ok(()) +} + +const ITEM_COUNT: usize = 1; +const BATCHES: usize = 10; + +#[test] +fn snapshot_disk_point_reads() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for batch in 0..BATCHES { + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, format!("abc{batch}").as_bytes(), seqno.next()); + } + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + + let item = tree.get(key, SeqNo::MAX)?.expect("should exist"); + assert_eq!("abc9".as_bytes(), &*item); + } + + let snapshot_seqno = seqno.get(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, tree.len(snapshot_seqno, None)?); + + // This batch will be too new for snapshot (invisible) + for batch in 0..BATCHES { + let batch_seqno = seqno.next(); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, format!("def{batch}").as_bytes(), batch_seqno); + } + } + tree.flush_active_memtable(0)?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + + let item = tree.get(key, snapshot_seqno)?.expect("should exist"); + assert_eq!("abc9".as_bytes(), &*item); + + let item = tree.get(key, SeqNo::MAX)?.expect("should exist"); + assert_eq!("def9".as_bytes(), &*item); + } + + Ok(()) +} + +#[test] +fn snapshot_disk_and_memtable_reads() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for batch in 0..BATCHES { + let batch_seqno = seqno.next(); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, format!("abc{batch}").as_bytes(), batch_seqno); + } + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + + let snapshot_seqno = seqno.get(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, tree.len(snapshot_seqno, None)?); + + // This batch will be in memtable and too new for snapshot (invisible) + for batch in 0..BATCHES { + let batch_seqno = seqno.next(); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, format!("def{batch}").as_bytes(), batch_seqno); + } + } + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + + let item = tree.get(key, snapshot_seqno)?.expect("should exist"); + assert_eq!("abc9".as_bytes(), &*item); + + let item = tree.get(key, SeqNo::MAX)?.expect("should exist"); + assert_eq!("def9".as_bytes(), &*item); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/snapshot_zombie.rs b/crates/lsm-tree/tests/snapshot_zombie.rs new file mode 100644 index 000000000..1605051ac --- /dev/null +++ b/crates/lsm-tree/tests/snapshot_zombie.rs @@ -0,0 +1,103 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +const ITEM_COUNT: usize = 5; + +#[test] +fn snapshot_zombie_memtable() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!(tree.iter(SeqNo::MAX, None).rev().count(), ITEM_COUNT); + + { + let snapshot_seqno = seqno.get(); + assert_eq!(ITEM_COUNT, tree.len(snapshot_seqno, None)?); + assert_eq!(ITEM_COUNT, tree.iter(snapshot_seqno, None).rev().count()); + } + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.remove(key, seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).rev().count(), 0); + + { + let snapshot_seqno = seqno.get(); + assert_eq!(0, tree.len(snapshot_seqno, None)?); + assert_eq!(0, tree.iter(snapshot_seqno, None).rev().count()); + } + + Ok(()) +} + +#[test] +fn snapshot_zombie_table() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + { + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "abc".as_bytes(), seqno.next()); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!(tree.iter(SeqNo::MAX, None).rev().count(), ITEM_COUNT); + + { + let snapshot_seqno = seqno.get(); + assert_eq!(ITEM_COUNT, tree.len(snapshot_seqno, None)?); + assert_eq!(ITEM_COUNT, tree.iter(snapshot_seqno, None).rev().count()); + } + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.remove(key, seqno.next()); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).rev().count(), 0); + + { + let snapshot_seqno = seqno.get(); + assert_eq!(0, tree.len(snapshot_seqno, None)?); + assert_eq!(0, tree.iter(snapshot_seqno, None).rev().count()); + assert_eq!(0, tree.prefix(b"", snapshot_seqno, None).count()); + } + } + + { + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).rev().count(), 0); + + { + let snapshot_seqno = seqno.get(); + assert_eq!(0, tree.len(snapshot_seqno, None)?); + assert_eq!(0, tree.iter(snapshot_seqno, None).rev().count()); + assert_eq!(0, tree.prefix(b"", snapshot_seqno, None).count()); + } + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/static_iterators.rs b/crates/lsm-tree/tests/static_iterators.rs new file mode 100644 index 000000000..83b7e0a91 --- /dev/null +++ b/crates/lsm-tree/tests/static_iterators.rs @@ -0,0 +1,363 @@ +// Copyright (c) 2024-present, fjall-rs +// This source code is licensed under both the Apache 2.0 and MIT License +// (found in the LICENSE-* files in the repository) + +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SequenceNumberCounter}; +use test_log::test; + +/// Test that iterators can be stored in a struct (proving they're 'static) +#[test] +fn static_iterator_ownership() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "value_a", 0); + tree.insert("b", "value_b", 1); + tree.insert("c", "value_c", 2); + + // Create an iterator and store it + let iter = tree.range("a"..="c", 3, None); + + // Drop the tree - the iterator should still own all necessary data + drop(tree); + + // Collect results from the iterator + let results: Vec<_> = iter + .map(|guard| guard.into_inner()) + .collect::>>()?; + + assert_eq!(results.len(), 3); + assert_eq!(results[0].0, b"a"); + assert_eq!(results[0].1, b"value_a"); + assert_eq!(results[1].0, b"b"); + assert_eq!(results[1].1, b"value_b"); + assert_eq!(results[2].0, b"c"); + assert_eq!(results[2].1, b"value_c"); + + Ok(()) +} + +/// Test that iterator can be moved across threads +#[test] +fn static_iterator_send_to_thread() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..100 { + tree.insert(format!("key_{:03}", i), format!("value_{}", i), i as u64); + } + + // Create iterator and move to another thread + let iter = tree.range("key_000"..="key_099", 100, None); + drop(tree); + + let handle = std::thread::spawn(move || { + iter.map(|guard| guard.key()) + .collect::>>() + .unwrap() + .len() + }); + + let count = handle.join().unwrap(); + assert_eq!(count, 100); + + Ok(()) +} + +/// Test static iterator with prefix +#[test] +fn static_iterator_prefix() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("user:1:name", "Alice", 0); + tree.insert("user:1:age", "30", 1); + tree.insert("user:2:name", "Bob", 2); + tree.insert("user:2:age", "25", 3); + tree.insert("other:data", "xyz", 4); + + let iter = tree.prefix("user:1", 5, None); + drop(tree); + + let results: Vec<_> = iter + .map(|guard| guard.key()) + .collect::>>()?; + + assert_eq!(results.len(), 2); + assert!(results[0].starts_with(b"user:1")); + assert!(results[1].starts_with(b"user:1")); + + Ok(()) +} + +/// Test reverse iteration with static iterator +#[test] +fn static_iterator_reverse() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..10 { + tree.insert(format!("key_{}", i), format!("value_{}", i), i as u64); + } + + let iter = tree.range("key_0"..="key_9", 10, None); + drop(tree); + + let results: Vec<_> = iter + .rev() + .map(|guard| guard.key()) + .collect::>>()?; + + assert_eq!(results.len(), 10); + // Should be in reverse order + assert_eq!(results[0], b"key_9"); + assert_eq!(results[9], b"key_0"); + + Ok(()) +} + +/// Test iterator with flushed segments (disk-based data) +#[test] +fn static_iterator_with_segments() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Write data and flush to disk + for i in 0..50 { + tree.insert(format!("seg_{:03}", i), format!("val_{}", i), i as u64); + } + tree.flush_active_memtable(50)?; + + // Write more data in memory + for i in 50..100 { + tree.insert(format!("seg_{:03}", i), format!("val_{}", i), i as u64); + } + + // Create iterator that spans both memtable and segments + let iter = tree.range("seg_000"..="seg_099", 100, None); + drop(tree); + + let count = iter + .map(|guard| guard.key()) + .collect::>>()? + .len(); + + assert_eq!(count, 100); + + Ok(()) +} + +/// Test BlobTree static iterator +#[test] +fn static_iterator_blob_tree() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + // Insert values that will be separated to blob files + let large_value = vec![b'X'; 2048]; + for i in 0..20 { + tree.insert(format!("blob_{:03}", i), large_value.clone(), i as u64); + } + + tree.flush_active_memtable(20)?; + + let iter = tree.range("blob_000"..="blob_019", 20, None); + drop(tree); + + let results: Vec<_> = iter + .map(|guard| guard.into_inner()) + .collect::>>()?; + + assert_eq!(results.len(), 20); + for (_, value) in results { + assert_eq!(value.len(), 2048); + } + + Ok(()) +} + +/// Test that iterator sees a consistent snapshot +#[test] +fn static_iterator_snapshot_isolation() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Initial data + for i in 0..10 { + tree.insert(format!("key_{}", i), "v1", i as u64); + } + + // Create iterator at seqno 10 + let iter = tree.range("key_0"..="key_9", 10, None); + + // Modify data after iterator creation + for i in 0..10 { + tree.insert(format!("key_{}", i), "v2", (10 + i) as u64); + } + + // Iterator should see old version (v1) + for guard in iter { + let (_, value) = guard.into_inner()?; + assert_eq!(value, b"v1"); + } + + // New iterator should see new version (v2) + let new_iter = tree.range("key_0"..="key_9", 20, None); + for guard in new_iter { + let (_, value) = guard.into_inner()?; + assert_eq!(value, b"v2"); + } + + Ok(()) +} + +/// Test multiple iterators alive at the same time +#[test] +fn static_iterator_multiple_concurrent() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..100 { + tree.insert(format!("key_{:03}", i), format!("val_{}", i), i as u64); + } + + // Create multiple iterators + let iter1 = tree.range("key_000"..="key_024", 100, None); + let iter2 = tree.range("key_025"..="key_049", 100, None); + let iter3 = tree.range("key_050"..="key_074", 100, None); + let iter4 = tree.range("key_075"..="key_099", 100, None); + + drop(tree); + + // Collect all results + let count1 = iter1.count(); + let count2 = iter2.count(); + let count3 = iter3.count(); + let count4 = iter4.count(); + + assert_eq!(count1, 25); + assert_eq!(count2, 25); + assert_eq!(count3, 25); + assert_eq!(count4, 25); + + Ok(()) +} + +/// Test that iterator properly maintains Version reference preventing data loss +#[test] +fn static_iterator_prevents_data_loss() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // Create data and flush + for i in 0..50 { + tree.insert(format!("key_{:03}", i), format!("val_{}", i), i as u64); + } + tree.flush_active_memtable(50)?; + + // Create iterator + let iter = tree.range("key_000"..="key_049", 50, None); + + // Even if we trigger compaction, the iterator should keep the segments alive + // (In practice, segments would be marked for deletion after compaction but + // the Version reference in the iterator prevents actual deletion) + + // Verify all data is accessible + let count = iter.count(); + assert_eq!(count, 50); + + Ok(()) +} + +/// Test iterator with tombstones +#[test] +fn static_iterator_with_tombstones() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for i in 0..20 { + tree.insert(format!("key_{:02}", i), format!("val_{}", i), i as u64); + } + + // Delete some keys + for i in 5..15 { + tree.remove(format!("key_{:02}", i), (20 + i) as u64); + } + + let iter = tree.range("key_00"..="key_19", 35, None); + drop(tree); + + let results: Vec<_> = iter + .map(|guard| guard.key()) + .collect::>>()?; + + // Should only see 10 keys (0-4 and 15-19) + assert_eq!(results.len(), 10); + assert_eq!(results[0], b"key_00"); + assert_eq!(results[4], b"key_04"); + assert_eq!(results[5], b"key_15"); + assert_eq!(results[9], b"key_19"); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/table_full_file_checksum.rs b/crates/lsm-tree/tests/table_full_file_checksum.rs new file mode 100644 index 000000000..c8f328a8c --- /dev/null +++ b/crates/lsm-tree/tests/table_full_file_checksum.rs @@ -0,0 +1,115 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; +use xxhash_rust::xxh3::xxh3_128; + +#[test] +fn table_full_file_checksum() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for key in ('a'..='z').map(|c| c.to_string()) { + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + let version = tree.current_version(); + let table = version.iter_tables().next().unwrap(); + + let expected_checksum = table.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(&*table.path)?); + assert_eq!( + real_checksum, expected_checksum, + "full file checksum mismatch", + ); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let version = tree.current_version(); + let table = version.iter_tables().next().unwrap(); + + let expected_checksum = table.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(&*table.path)?); + assert_eq!( + real_checksum, expected_checksum, + "full file checksum mismatch", + ); + } + + Ok(()) +} + +#[test] +fn table_full_file_detect_corruption() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for key in ('a'..='z').map(|c| c.to_string()) { + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + let version = tree.current_version(); + let table = version.iter_tables().next().unwrap(); + + let expected_checksum = table.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(&*table.path)?); + assert_eq!( + real_checksum, expected_checksum, + "full file checksum mismatch", + ); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let version = tree.current_version(); + let table = version.iter_tables().next().unwrap(); + + { + use std::io::{Seek, Write}; + + let mut f = std::fs::OpenOptions::new().write(true).open(&*table.path)?; + + f.seek(std::io::SeekFrom::Start(100))?; + f.write_all(b"!")?; + f.sync_all()?; + } + + let expected_checksum = table.checksum().into_u128(); + let real_checksum = xxh3_128(&std::fs::read(&*table.path)?); + assert_ne!( + real_checksum, expected_checksum, + "full file checksum should not match", + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/table_point_reads.rs b/crates/lsm-tree/tests/table_point_reads.rs new file mode 100644 index 000000000..95e59cd17 --- /dev/null +++ b/crates/lsm-tree/tests/table_point_reads.rs @@ -0,0 +1,172 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +const ITEM_COUNT: usize = 1_000; + +#[test] +fn table_point_reads() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + assert!(tree.contains_key(key, SeqNo::MAX)?, "{key:?} not found"); + } + + Ok(()) +} + +#[test] +fn table_point_reads_mvcc() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + tree.insert(key, "0", 0); + tree.insert(key, "1", 1); + tree.insert(key, "2", 2); + } + tree.flush_active_memtable(0)?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + + let item = tree.get_internal_entry(&key, SeqNo::MAX)?.unwrap(); + assert_eq!(item.key.seqno, 2); + assert_eq!(&*item.value, b"2"); + + let item = tree.get_internal_entry(&key, 3)?.unwrap(); + assert_eq!(&*item.value, b"2"); + + let item = tree.get_internal_entry(&key, 2)?.unwrap(); + assert_eq!(&*item.value, b"1"); + + let item = tree.get_internal_entry(&key, 1)?.unwrap(); + assert_eq!(&*item.value, b"0"); + } + + Ok(()) +} + +#[test] +fn table_point_reads_mvcc_slab() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + let keys = [0, 1, 2] + .into_iter() + .map(u64::to_be_bytes) + .collect::>(); + + for key in &keys { + for seqno in 0..ITEM_COUNT as u64 { + tree.insert(key, seqno.to_string(), seqno); + } + } + tree.flush_active_memtable(0)?; + + for key in &keys { + let item = tree.get_internal_entry(key, SeqNo::MAX)?.unwrap(); + assert_eq!(item.key.seqno, ITEM_COUNT as u64 - 1); + } + + for key in &keys { + // NOTE: Need to start at seqno=1 + for seqno in 1..ITEM_COUNT as u64 { + let item = tree.get_internal_entry(key, seqno)?.unwrap(); + + // NOTE: When snapshot is =1, it will read any items with + // seqno less than 1 + assert_eq!( + String::from_utf8_lossy(&item.value) + .parse::() + .unwrap(), + seqno - 1 + ); + } + } + + Ok(()) +} + +#[test] +fn blob_tree_table_point_reads_mvcc_slab() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let keys = [0, 1, 2] + .into_iter() + .map(u64::to_be_bytes) + .collect::>(); + + for key in &keys { + for seqno in 0..ITEM_COUNT as u64 { + tree.insert(key, seqno.to_string(), seqno); + } + } + tree.flush_active_memtable(0)?; + + for key in &keys { + let item = tree.get(key, SeqNo::MAX)?.unwrap(); + assert_eq!( + String::from_utf8_lossy(&item).parse::().unwrap(), + ITEM_COUNT as u64 - 1 + ); + } + + for key in &keys { + // NOTE: Need to start at seqno=1 + for seqno in 1..ITEM_COUNT as u64 { + let value = tree.get(key, seqno)?.unwrap(); + + // NOTE: When snapshot is =1, it will read any items with + // seqno less than 1 + assert_eq!( + String::from_utf8_lossy(&value).parse::().unwrap(), + seqno - 1 + ); + } + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/table_range.rs b/crates/lsm-tree/tests/table_range.rs new file mode 100644 index 000000000..dddf7e72c --- /dev/null +++ b/crates/lsm-tree/tests/table_range.rs @@ -0,0 +1,157 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, Guard, SeqNo, + SequenceNumberCounter, +}; +use test_log::test; + +const ITEM_COUNT: usize = 1_000_000; + +#[test] +fn a_lot_of_ranges() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + let iter = tree.range( + 1_000u64.to_be_bytes()..11_000u64.to_be_bytes(), + SeqNo::MAX, + None, + ); + assert_eq!(10_000, iter.count()); + + let iter = tree.range( + 1_000u64.to_be_bytes()..11_000u64.to_be_bytes(), + SeqNo::MAX, + None, + ); + assert_eq!(10_000, iter.rev().count()); + + let mut iter = tree.range( + 1_000u64.to_be_bytes()..11_000u64.to_be_bytes(), + SeqNo::MAX, + None, + ); + let mut count = 0; + + for x in 0.. { + if x % 2 == 0 { + let Some(_) = iter.next() else { + break; + }; + + count += 1; + } else { + let Some(_) = iter.next_back() else { + break; + }; + + count += 1; + } + } + + assert_eq!(10_000, count); + + Ok(()) +} + +#[test] +fn table_range_last_back() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + let value = (0..2_000).map(|_| 0).collect::>(); + + for x in 0..10_u64 { + let key = x.to_be_bytes(); + tree.insert(key, &value, 0); + } + tree.flush_active_memtable(0)?; + + let iter = tree.range(0u64.to_be_bytes()..10u64.to_be_bytes(), SeqNo::MAX, None); + assert_eq!(10, iter.count()); + + let iter = tree.range(0u64.to_be_bytes()..10u64.to_be_bytes(), SeqNo::MAX, None); + assert_eq!(10, iter.rev().count()); + + let mut iter = tree.range(0u64.to_be_bytes()..5u64.to_be_bytes(), SeqNo::MAX, None); + + assert_eq!(0u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(1u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(2u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(3u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(4u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert!(iter.next_back().is_none()); + + Ok(()) +} + +#[test] +fn table_range_last_back_2() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + let value = (0..2_000).map(|_| 0).collect::>(); + + for x in 0..10_u64 { + let key = x.to_be_bytes(); + tree.insert(key, &value, 0); + } + tree.insert(10u64.to_be_bytes(), [], 0); + tree.insert(11u64.to_be_bytes(), [], 0); + tree.flush_active_memtable(0)?; + + let iter = tree.range(0u64.to_be_bytes()..10u64.to_be_bytes(), SeqNo::MAX, None); + assert_eq!(10, iter.count()); + + let iter = tree.range(0u64.to_be_bytes()..10u64.to_be_bytes(), SeqNo::MAX, None); + assert_eq!(10, iter.rev().count()); + + let mut iter = tree.range(0u64.to_be_bytes()..12u64.to_be_bytes(), SeqNo::MAX, None); + + assert_eq!(0u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(1u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(2u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(3u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(4u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(5u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(6u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(7u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(8u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(9u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(10u64.to_be_bytes(), &*iter.next().unwrap().key()?); + assert_eq!(11u64.to_be_bytes(), &*iter.next_back().unwrap().key()?); + assert!(iter.next().is_none()); + assert!(iter.next_back().is_none()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/table_range_oob.rs b/crates/lsm-tree/tests/table_range_oob.rs new file mode 100644 index 000000000..5082176ee --- /dev/null +++ b/crates/lsm-tree/tests/table_range_oob.rs @@ -0,0 +1,87 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +const ITEM_COUNT: usize = 100; + +#[test] +fn table_range_out_of_bounds_lo() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for key in ('h'..='o').map(|c| c.to_string()) { + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + assert_eq!(4, tree.range(..="k", SeqNo::MAX, None).count()); + assert_eq!(4, tree.range(..="k", SeqNo::MAX, None).rev().count()); + + assert_eq!(4, tree.range("0"..="k", SeqNo::MAX, None).count()); + assert_eq!(4, tree.range("0"..="k", SeqNo::MAX, None).rev().count()); + + Ok(()) +} + +#[test] +fn table_range_out_of_bounds_hi() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + tree.flush_active_memtable(0)?; + + assert_eq!( + 50, + tree.range((50u64.to_be_bytes()).., SeqNo::MAX, None) + .count() + ); + assert_eq!( + 50, + tree.range((50u64.to_be_bytes()).., SeqNo::MAX, None) + .rev() + .count() + ); + + assert_eq!( + 50, + tree.range( + (50u64.to_be_bytes())..(150u64.to_be_bytes()), + SeqNo::MAX, + None + ) + .count() + ); + assert_eq!( + 50, + tree.range( + (50u64.to_be_bytes())..(150u64.to_be_bytes()), + SeqNo::MAX, + None + ) + .rev() + .count() + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/table_remove_weak.rs b/crates/lsm-tree/tests/table_remove_weak.rs new file mode 100644 index 000000000..59dabd8b1 --- /dev/null +++ b/crates/lsm-tree/tests/table_remove_weak.rs @@ -0,0 +1,28 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn table_remove_weak_simple() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("a", "b", 1); + tree.remove_weak("a", 2); + + tree.flush_active_memtable(0)?; + + assert!(tree.get("a", SeqNo::MAX)?.is_none()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/table_weak_tombstones.rs b/crates/lsm-tree/tests/table_weak_tombstones.rs new file mode 100644 index 000000000..b49ef8fab --- /dev/null +++ b/crates/lsm-tree/tests/table_weak_tombstones.rs @@ -0,0 +1,75 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; + +#[test] +fn weak_tombstone_counts_single_pair() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + folder.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert(b"a", b"old", 1); + tree.remove_weak(b"a", 2); + tree.flush_active_memtable(0)?; + + assert_eq!(tree.weak_tombstone_count(), 1); + assert_eq!(tree.weak_tombstone_reclaimable_count(), 1); + + Ok(()) +} + +#[test] +fn weak_tombstone_counts_multiple_keys() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + folder.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert(b"a", b"old", 10); + tree.remove_weak(b"a", 11); + + tree.remove_weak(b"b", 12); + + tree.insert(b"c", b"old", 13); + tree.insert(b"c", b"new", 14); + tree.remove_weak(b"c", 15); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.weak_tombstone_count(), 3); + assert_eq!(tree.weak_tombstone_reclaimable_count(), 2); + + Ok(()) +} + +#[test] +fn weak_tombstone_counts_multiple_weak() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + folder.path(), + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert(b"a", b"old", 10); + tree.remove_weak(b"a", 11); + tree.remove_weak(b"a", 12); + tree.remove_weak(b"a", 13); + tree.remove_weak(b"a", 14); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.weak_tombstone_count(), 4); + assert_eq!(tree.weak_tombstone_reclaimable_count(), 1); // a:10 is paired with a:11 + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_approx_len.rs b/crates/lsm-tree/tests/tree_approx_len.rs new file mode 100644 index 000000000..bf770fbb2 --- /dev/null +++ b/crates/lsm-tree/tests/tree_approx_len.rs @@ -0,0 +1,212 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_approx_len_sealed() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 0); + + tree.insert("a", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 1); + + tree.insert("b", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 2); + + let _ = tree.rotate_memtable().unwrap(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 2); + + Ok(()) +} + +#[test] +fn tree_approx_len_sealed_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 0); + + tree.insert("a", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 1); + + tree.insert("b", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 2); + + let _ = tree.rotate_memtable().unwrap(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 2); + + Ok(()) +} + +#[test] +fn tree_approx_len() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 0); + + tree.insert("a", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 1); + + tree.insert("b", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 2); + + tree.insert("a", "", 1); + + // Approximate count diverges + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 3); + + tree.remove("a", 2); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 4); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 4); + + tree.remove("b", 4); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 5); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 5); + + tree.major_compact(u64::MAX, 5)?; + + // Approximate count converges + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 0); + + Ok(()) +} + +#[test] +fn tree_approx_len_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 0); + + tree.insert("a", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 1); + + tree.insert("b", "", 0); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 2); + + tree.insert("a", "", 1); + + // Approximate count diverges + assert_eq!(tree.len(SeqNo::MAX, None)?, 2); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 3); + + tree.remove("a", 2); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 4); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert!(!tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 4); + + tree.remove("b", 4); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 5); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 5); + + tree.major_compact(u64::MAX, 5)?; + + // Approximate count converges + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!(tree.approximate_len(), 0); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_bulk_ingest.rs b/crates/lsm-tree/tests/tree_bulk_ingest.rs new file mode 100644 index 000000000..3e8e51a4a --- /dev/null +++ b/crates/lsm-tree/tests/tree_bulk_ingest.rs @@ -0,0 +1,205 @@ +use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, Guard, KvSeparationOptions, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +const ITEM_COUNT: usize = 100_000; + +#[test] +fn tree_bulk_ingest() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), visible_seqno.clone()).open()?; + + let mut ingestion = tree.ingestion()?; + for x in 0..ITEM_COUNT as u64 { + let k = x.to_be_bytes(); + let v = nanoid::nanoid!(); + ingestion.write(k, v)?; + } + ingestion.finish()?; + + assert_eq!(visible_seqno.get(), seqno.get()); + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT, + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT, + ); + + Ok(()) +} + +#[test] +fn tree_copy() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let src = Config::new(&folder, seqno.clone(), visible_seqno.clone()).open()?; + + let mut ingestion = src.ingestion()?; + for x in 0..ITEM_COUNT as u64 { + let k = x.to_be_bytes(); + let v = nanoid::nanoid!(); + ingestion.write(k, v)?; + } + ingestion.finish()?; + + assert_eq!(visible_seqno.get(), seqno.get()); + + assert_eq!(src.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + src.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT, + ); + assert_eq!( + src.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT, + ); + + let folder = get_tmp_folder(); + let dest = Config::new(&folder, seqno.clone(), visible_seqno.clone()).open()?; + + let mut ingestion = dest.ingestion()?; + for item in src.iter(SeqNo::MAX, None) { + let (k, v) = item.into_inner().unwrap(); + ingestion.write(k, v)?; + } + ingestion.finish()?; + + assert_eq!(visible_seqno.get(), seqno.get()); + + assert_eq!(dest.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + dest.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT, + ); + assert_eq!( + dest.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT, + ); + + Ok(()) +} + +#[test] +fn blob_tree_bulk_ingest() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), visible_seqno.clone()) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let mut ingestion = tree.ingestion()?; + for x in 0..ITEM_COUNT as u64 { + let k = x.to_be_bytes(); + let v = nanoid::nanoid!(); + ingestion.write(k, v)?; + } + ingestion.finish()?; + + assert_eq!(visible_seqno.get(), seqno.get()); + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT, + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT, + ); + assert_eq!(1, tree.blob_file_count()); + + Ok(()) +} + +#[test] +fn blob_tree_copy() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let src = Config::new(&folder, seqno.clone(), visible_seqno.clone()) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let mut ingestion = src.ingestion()?; + for x in 0..ITEM_COUNT as u64 { + let k = x.to_be_bytes(); + let v = nanoid::nanoid!(); + ingestion.write(k, v)?; + } + ingestion.finish()?; + + assert_eq!(visible_seqno.get(), seqno.get()); + + assert_eq!(src.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + src.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT, + ); + assert_eq!( + src.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT, + ); + assert_eq!(1, src.blob_file_count()); + + let folder = get_tmp_folder(); + let dest = Config::new(&folder, seqno.clone(), visible_seqno.clone()) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + let mut ingestion = dest.ingestion()?; + for item in src.iter(SeqNo::MAX, None) { + let (k, v) = item.into_inner().unwrap(); + ingestion.write(k, v)?; + } + ingestion.finish()?; + + assert_eq!(visible_seqno.get(), seqno.get()); + + assert_eq!(dest.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + dest.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT, + ); + assert_eq!( + dest.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT, + ); + assert_eq!(1, dest.blob_file_count()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_clear.rs b/crates/lsm-tree/tests/tree_clear.rs new file mode 100644 index 000000000..022a8db48 --- /dev/null +++ b/crates/lsm-tree/tests/tree_clear.rs @@ -0,0 +1,43 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_clear() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), visible_seqno.clone()).open()?; + + assert_eq!(0, tree.len(visible_seqno.get(), None)?); + + { + let seqno = seqno.next(); + tree.insert("a", "a", seqno); + visible_seqno.fetch_max(seqno + 1); + } + + assert!(tree.contains_key("a", SeqNo::MAX)?); + assert_eq!(1, tree.len(visible_seqno.get(), None)?); + + tree.clear()?; + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert_eq!(0, tree.len(visible_seqno.get(), None)?); + + { + let seqno = seqno.next(); + tree.insert("a", "a", seqno); + visible_seqno.fetch_max(seqno + 1); + } + + tree.flush_active_memtable(0)?; + assert!(tree.contains_key("a", SeqNo::MAX)?); + assert_eq!(1, tree.len(visible_seqno.get(), None)?); + + tree.clear()?; + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert_eq!(0, tree.len(visible_seqno.get(), None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_clear_snapshot.rs b/crates/lsm-tree/tests/tree_clear_snapshot.rs new file mode 100644 index 000000000..21342f5f2 --- /dev/null +++ b/crates/lsm-tree/tests/tree_clear_snapshot.rs @@ -0,0 +1,30 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_clear_snapshot() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + let visible_seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), visible_seqno.clone()).open()?; + + assert_eq!(0, tree.len(visible_seqno.get(), None)?); + + for c in ["a", "b", "c"] { + let seqno = seqno.next(); + tree.insert(c, c, seqno); + visible_seqno.fetch_max(seqno + 1); + } + assert_eq!(3, tree.len(visible_seqno.get(), None)?); + + let snapshot = visible_seqno.get(); + + tree.clear()?; + assert_eq!(0, tree.len(visible_seqno.get(), None)?); + + assert_eq!(3, tree.len(snapshot, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_count.rs b/crates/lsm-tree/tests/tree_count.rs new file mode 100644 index 000000000..01a8c6de8 --- /dev/null +++ b/crates/lsm-tree/tests/tree_count.rs @@ -0,0 +1,154 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter, Slice}; +use test_log::test; + +const ITEM_COUNT: usize = 1_000; + +#[test] +fn tree_memtable_count() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT + ); + + Ok(()) +} + +#[test] +fn tree_flushed_count() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT + ); + + Ok(()) +} + +#[test] +fn tree_flushed_count_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), 0); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT + ); + + Ok(()) +} + +#[test] +fn tree_non_locking_count() -> lsm_tree::Result<()> { + use std::ops::Bound::{self, Excluded, Unbounded}; + + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = "a"; + tree.insert(key, value.as_bytes(), x); + } + + tree.flush_active_memtable(0)?; + + let mut range: (Bound, Bound) = (Unbounded, Unbounded); + let mut count = 0; + + loop { + let chunk = tree + .range(range.clone(), SeqNo::MAX, None) + .map(|x| x.into_inner()) + .take(10) + .collect::>>()?; + + if chunk.is_empty() { + break; + } + + count += chunk.len(); + + let (key, _) = chunk.last().unwrap(); + range = (Excluded(key.clone()), Unbounded); + } + + assert_eq!(count, ITEM_COUNT); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_delete_loop.rs b/crates/lsm-tree/tests/tree_delete_loop.rs new file mode 100644 index 000000000..fe29a801e --- /dev/null +++ b/crates/lsm-tree/tests/tree_delete_loop.rs @@ -0,0 +1,94 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_delete_by_prefix() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 10_000; + + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let seqno = SequenceNumberCounter::default(); + + for x in 0..ITEM_COUNT as u64 { + let value = "old".as_bytes(); + let batch_seqno = seqno.next(); + + tree.insert(format!("a:{x}").as_bytes(), value, batch_seqno); + tree.insert(format!("b:{x}").as_bytes(), value, batch_seqno); + tree.insert(format!("c:{x}").as_bytes(), value, batch_seqno); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 3); + assert_eq!( + tree.prefix("a:".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert_eq!( + tree.prefix("b:".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert_eq!( + tree.prefix("c:".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + + for item in tree.prefix("b:".as_bytes(), SeqNo::MAX, None) { + let key = item.key()?; + tree.remove(key, seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("a:".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert_eq!(tree.prefix("b:".as_bytes(), SeqNo::MAX, None).count(), 0); + assert_eq!( + tree.prefix("c:".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + + Ok(()) +} + +#[test] +fn tree_delete_by_range() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let value = "old".as_bytes(); + tree.insert("a".as_bytes(), value, 0); + tree.insert("b".as_bytes(), value, 0); + tree.insert("c".as_bytes(), value, 0); + tree.insert("d".as_bytes(), value, 0); + tree.insert("e".as_bytes(), value, 0); + tree.insert("f".as_bytes(), value, 0); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 6); + + for item in tree.range("c"..="e", SeqNo::MAX, None) { + let key = item.key()?; + tree.remove(key, 1); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, 3); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_different_block_size.rs b/crates/lsm-tree/tests/tree_different_block_size.rs new file mode 100644 index 000000000..9cb2137f3 --- /dev/null +++ b/crates/lsm-tree/tests/tree_different_block_size.rs @@ -0,0 +1,72 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +const ITEM_COUNT: usize = 1_000; + +#[test] +fn tree_block_size_after_recovery() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(2_048)) + // .index_block_size_policy(BlockSizePolicy::all(2_048)) + .open()?; + + let seqno = SequenceNumberCounter::default(); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), seqno.next()); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(ITEM_COUNT, tree.len(SeqNo::MAX, None)?); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(2_048)) + // .index_block_size_policy(BlockSizePolicy::all(2_048)) + .open()?; + assert_eq!(ITEM_COUNT, tree.len(SeqNo::MAX, None)?); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(4_096)) + // .index_block_size_policy(BlockSizePolicy::all(4_096)) + .open()?; + assert_eq!(ITEM_COUNT, tree.len(SeqNo::MAX, None)?); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(78_652)) + // .index_block_size_policy(BlockSizePolicy::all(78_652)) + .open()?; + assert_eq!(ITEM_COUNT, tree.len(SeqNo::MAX, None)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_disjoint_iter.rs b/crates/lsm-tree/tests/tree_disjoint_iter.rs new file mode 100644 index 000000000..a53a3afad --- /dev/null +++ b/crates/lsm-tree/tests/tree_disjoint_iter.rs @@ -0,0 +1,74 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter, Slice}; +use test_log::test; + +macro_rules! iter_closed { + ($iter:expr) => { + assert!($iter.next().is_none(), "iterator should be closed (done)"); + assert!( + $iter.next_back().is_none(), + "iterator should be closed (done)" + ); + }; +} + +#[test] +fn tree_disjoint_iter() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = crate::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // IMPORTANT: Purposefully mangle the order of IDs + // to make sure stuff is still getting read in the correct order + // even if written out of order + let ids = [["f", "e", "d"], ["a", "b", "c"]]; + + for batch in ids { + for id in batch { + tree.insert(id, vec![], 0); + } + tree.flush_active_memtable(0)?; + } + + // NOTE: Forwards + + let mut iter = tree.iter(SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Reverse + + let mut iter = tree.iter(SeqNo::MAX, None).rev(); + + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"d"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Ping Pong + + let mut iter = tree.iter(SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"a"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next_back().unwrap().key()?); + assert_eq!(Slice::from(*b"b"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"e"), iter.next_back().unwrap().key()?); + assert_eq!(Slice::from(*b"c"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"d"), iter.next_back().unwrap().key()?); + iter_closed!(iter); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_disjoint_point_read.rs b/crates/lsm-tree/tests/tree_disjoint_point_read.rs new file mode 100644 index 000000000..895fe6615 --- /dev/null +++ b/crates/lsm-tree/tests/tree_disjoint_point_read.rs @@ -0,0 +1,71 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn tree_disjoint_point_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + + tree.flush_active_memtable(0)?; + + tree.insert("d", "d", 0); + tree.insert("e", "e", 0); + tree.insert("f", "f", 0); + + tree.flush_active_memtable(0)?; + + for key in [b"a", b"b", b"c", b"d", b"e", b"f"] { + let value = tree.get(key, SeqNo::MAX).unwrap().unwrap(); + assert_eq!(&*value, key) + } + + Ok(()) +} + +#[test] +fn tree_disjoint_point_read_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + + tree.flush_active_memtable(0)?; + + tree.insert("d", "d", 0); + tree.insert("e", "e", 0); + tree.insert("f", "f", 0); + + tree.flush_active_memtable(0)?; + + for key in [b"a", b"b", b"c", b"d", b"e", b"f"] { + let value = tree.get(key, SeqNo::MAX).unwrap().unwrap(); + assert_eq!(&*value, key) + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_disjoint_prefix.rs b/crates/lsm-tree/tests/tree_disjoint_prefix.rs new file mode 100644 index 000000000..fb1907b49 --- /dev/null +++ b/crates/lsm-tree/tests/tree_disjoint_prefix.rs @@ -0,0 +1,70 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter, Slice}; +use test_log::test; + +macro_rules! iter_closed { + ($iter:expr) => { + assert!($iter.next().is_none(), "iterator should be closed (done)"); + assert!( + $iter.next_back().is_none(), + "iterator should be closed (done)" + ); + }; +} + +#[test] +fn tree_disjoint_prefix() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = crate::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // IMPORTANT: Purposefully mangle the order of IDs + // to make sure stuff is still getting read in the correct order + // even if written out of order + let ids = [ + ["cc", "ca", "cb"], + ["aa", "ab", "ac"], + ["dc", "da", "db"], + ["ba", "bb", "bc"], + ]; + + for batch in ids { + for id in batch { + tree.insert(id, vec![], 0); + } + tree.flush_active_memtable(0)?; + } + + // NOTE: Forwards + + let mut iter = tree.prefix("d", SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"da"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"db"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"dc"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Reverse + + let mut iter = tree.prefix("d", SeqNo::MAX, None).rev(); + + assert_eq!(Slice::from(*b"dc"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"db"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"da"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Ping Pong + + let mut iter = tree.prefix("d", SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"da"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"dc"), iter.next_back().unwrap().key()?); + assert_eq!(Slice::from(*b"db"), iter.next().unwrap().key()?); + iter_closed!(iter); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_disjoint_range.rs b/crates/lsm-tree/tests/tree_disjoint_range.rs new file mode 100644 index 000000000..d2b81c086 --- /dev/null +++ b/crates/lsm-tree/tests/tree_disjoint_range.rs @@ -0,0 +1,106 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter, Slice}; +use test_log::test; + +macro_rules! iter_closed { + ($iter:expr) => { + assert!($iter.next().is_none(), "iterator should be closed (done)"); + assert!( + $iter.next_back().is_none(), + "iterator should be closed (done)" + ); + }; +} + +#[test] +fn tree_disjoint_range() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = crate::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + // IMPORTANT: Purposefully mangle the order of IDs + // to make sure stuff is still getting read in the correct order + // even if written out of order + let ids = [ + ["f", "e", "d"], + ["g", "h", "i"], + ["c", "b", "a"], + ["j", "k", "l"], + ]; + + for batch in ids { + for id in batch { + tree.insert(id, vec![], 0); + } + tree.flush_active_memtable(0)?; + } + + // NOTE: Forwards + + let mut iter = tree.range("e".."i", SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Forwards inclusive + + let mut iter = tree.range("e"..="i", SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Reverse + + let mut iter = tree.range("e".."i", SeqNo::MAX, None).rev(); + + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Reverse inclusive + + let mut iter = tree.range("e"..="i", SeqNo::MAX, None).rev(); + + assert_eq!(Slice::from(*b"i"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"h"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Ping Pong + + let mut iter = tree.range("e".."i", SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"h"), iter.next_back().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"g"), iter.next_back().unwrap().key()?); + iter_closed!(iter); + + // NOTE: Ping Pong inclusive + + let mut iter = tree.range("e"..="i", SeqNo::MAX, None); + + assert_eq!(Slice::from(*b"e"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"i"), iter.next_back().unwrap().key()?); + assert_eq!(Slice::from(*b"f"), iter.next().unwrap().key()?); + assert_eq!(Slice::from(*b"h"), iter.next_back().unwrap().key()?); + assert_eq!(Slice::from(*b"g"), iter.next().unwrap().key()?); + iter_closed!(iter); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_drop_range.rs b/crates/lsm-tree/tests/tree_drop_range.rs new file mode 100644 index 000000000..02cdd75de --- /dev/null +++ b/crates/lsm-tree/tests/tree_drop_range.rs @@ -0,0 +1,277 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, AnyTree, Config, SeqNo, SequenceNumberCounter}; +use std::ops::Bound::{Excluded, Included, Unbounded}; + +fn populate_tables(tree: &AnyTree) -> lsm_tree::Result<()> { + for key in 'a'..='e' { + tree.insert([key as u8], "", 0); + tree.flush_active_memtable(0)?; + } + Ok(()) +} + +#[test] +fn tree_drop_range_basic() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + assert_eq!(1, tree.l0_run_count()); + assert_eq!(5, tree.table_count()); + + tree.drop_range("a"..="c")?; + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert!(!tree.contains_key("b", SeqNo::MAX)?); + assert!(!tree.contains_key("c", SeqNo::MAX)?); + assert!(tree.contains_key("d", SeqNo::MAX)?); + assert!(tree.contains_key("e", SeqNo::MAX)?); + + assert_eq!(1, tree.l0_run_count()); + assert_eq!(2, tree.table_count()); + + Ok(()) +} + +#[test] +fn tree_drop_range_partial_table_overlap_kept() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for key in ['a', 'b', 'c', 'd', 'e'] { + tree.insert([key as u8], "", 0); + } + tree.flush_active_memtable(0)?; + + assert_eq!(1, tree.l0_run_count()); + assert_eq!(1, tree.table_count()); + + tree.drop_range("b".."d")?; + + for key in ['a', 'b', 'c', 'd', 'e'] { + assert!(tree.contains_key([key as u8], SeqNo::MAX)?); + } + + assert_eq!(1, tree.l0_run_count()); + assert_eq!(1, tree.table_count()); + + Ok(()) +} + +#[test] +fn tree_drop_range_upper_exclusive() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range("a".."d")?; + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert!(!tree.contains_key("b", SeqNo::MAX)?); + assert!(!tree.contains_key("c", SeqNo::MAX)?); + assert!(tree.contains_key("d", SeqNo::MAX)?); + assert!(tree.contains_key("e", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_lower_exclusive() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range::<&str, _>((Excluded("a"), Included("c")))?; + + assert!(tree.contains_key("a", SeqNo::MAX)?); + assert!(!tree.contains_key("b", SeqNo::MAX)?); + assert!(!tree.contains_key("c", SeqNo::MAX)?); + assert!(tree.contains_key("d", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_unbounded_lower_inclusive_upper() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range::<&str, _>((Unbounded, Included("c")))?; + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert!(!tree.contains_key("b", SeqNo::MAX)?); + assert!(!tree.contains_key("c", SeqNo::MAX)?); + assert!(tree.contains_key("d", SeqNo::MAX)?); + assert!(tree.contains_key("e", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_unbounded_lower_exclusive_upper() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range::<&str, _>((Unbounded, Excluded("d")))?; + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert!(!tree.contains_key("b", SeqNo::MAX)?); + assert!(!tree.contains_key("c", SeqNo::MAX)?); + assert!(tree.contains_key("d", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_exclusive_empty_interval() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range::<&str, _>((Excluded("b"), Excluded("b")))?; + + assert!(tree.contains_key("a", SeqNo::MAX)?); + assert!(tree.contains_key("b", SeqNo::MAX)?); + assert!(tree.contains_key("c", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_empty_tree() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.drop_range("a"..="c")?; + + assert_eq!(0, tree.l0_run_count()); + assert_eq!(0, tree.table_count()); + + Ok(()) +} + +#[test] +fn tree_drop_range_unbounded_upper() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range("c"..)?; + + assert!(tree.contains_key("a", SeqNo::MAX)?); + assert!(tree.contains_key("b", SeqNo::MAX)?); + assert!(!tree.contains_key("c", SeqNo::MAX)?); + assert!(!tree.contains_key("d", SeqNo::MAX)?); + assert!(!tree.contains_key("e", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_clear_all() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range::<&str, _>(..)?; + + assert_eq!(0, tree.l0_run_count()); + assert_eq!(0, tree.table_count()); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + assert!(!tree.contains_key("e", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_drop_range_inverted_bounds_is_noop() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + populate_tables(&tree)?; + + tree.drop_range("c".."a")?; + tree.drop_range("c"..="a")?; + + // All keys remain because the range is treated as empty. + for key in 'a'..='e' { + assert!(tree.contains_key([key as u8], SeqNo::MAX)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_filter_hit_rate.rs b/crates/lsm-tree/tests/tree_filter_hit_rate.rs new file mode 100644 index 000000000..281eb2e1c --- /dev/null +++ b/crates/lsm-tree/tests/tree_filter_hit_rate.rs @@ -0,0 +1,86 @@ +#[test_log::test] +#[cfg(feature = "metrics")] +fn tree_filter_hit_rate() -> lsm_tree::Result<()> { + use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; + + let a = { + let folder = get_tmp_folder(); + let path = folder.path(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(path, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for k in 0u64..10_000 { + tree.insert(k.to_be_bytes(), "abc", seqno.next()); + } + tree.flush_active_memtable(0)?; + tree.major_compact(u64::MAX, 0)?; + + for k in 0u64..10_000 { + tree.get(k.to_be_bytes(), SeqNo::MAX).unwrap().unwrap(); + } + + tree.metrics().filter_efficiency() + }; + + { + let folder = get_tmp_folder(); + let path = folder.path(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(path, seqno.clone(), SequenceNumberCounter::default()) + .with_kv_separation(Some(KvSeparationOptions::default().separation_threshold(1))) + .open()?; + + for k in 0u64..10_000 { + tree.insert(k.to_be_bytes(), "abc", seqno.next()); + } + tree.flush_active_memtable(0)?; + tree.major_compact(u64::MAX, 0)?; + + for k in 0u64..10_000 { + tree.get(k.to_be_bytes(), SeqNo::MAX).unwrap().unwrap(); + } + + assert_eq!(a, tree.metrics().filter_efficiency()); + } + + Ok(()) +} + +#[test_log::test] +#[cfg(feature = "metrics")] +fn tree_filter_hit_rate_hit() -> lsm_tree::Result<()> { + use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, KvSeparationOptions, SeqNo, SequenceNumberCounter, + }; + + let folder = get_tmp_folder(); + let path = folder.path(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(path, seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a", "a", 0); + tree.insert("c", "a", 0); + tree.flush_active_memtable(0)?; + + assert!(tree.get("b", SeqNo::MAX)?.is_none()); + assert_eq!(1, tree.metrics().filter_queries()); + assert_eq!(1, tree.metrics().io_skipped_by_filter()); + assert_eq!(1.0, tree.metrics().filter_efficiency()); + + assert!(tree.get("a", SeqNo::MAX)?.is_some()); + assert_eq!(1.0, tree.metrics().filter_efficiency()); + + assert!(tree.get("a", SeqNo::MAX)?.is_some()); + assert_eq!(1.0, tree.metrics().filter_efficiency()); + + assert!(tree.get("a", SeqNo::MAX)?.is_some()); + assert_eq!(1.0, tree.metrics().filter_efficiency()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_flush_eviction.rs b/crates/lsm-tree/tests/tree_flush_eviction.rs new file mode 100644 index 000000000..714092db7 --- /dev/null +++ b/crates/lsm-tree/tests/tree_flush_eviction.rs @@ -0,0 +1,138 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use std::sync::Arc; +use test_log::test; + +#[test] +fn tree_flush_eviction_1() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.remove_weak("a", 1); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + // NOTE: Should not evict weak tombstone + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn tree_flush_eviction_2() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.remove_weak("a", 1); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + // NOTE: Should evict old value, thus weak tombstone too + tree.flush_active_memtable(1)?; + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn tree_flush_eviction_3() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.remove("a", 1); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + // NOTE: Should evict old value, but tombstone should stay until last level + tree.flush_active_memtable(1)?; + assert_eq!(1, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + // NOTE: Should evict tombstone because last level + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 6)), 0)?; + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn tree_flush_eviction_4() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.remove("a", 1); + tree.insert("a", "a", 2); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + + // NOTE: Tombstone should stay because of seqno threshold + tree.flush_active_memtable(1)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + assert_eq!( + 1, + tree.current_version() + .level(0) + .expect("should exist") + .first() + .expect("should have at least 1 run") + .first() + .expect("should have one table") + .metadata + .tombstone_count + ); + + // NOTE: Should evict tombstone because last level + tree.compact(Arc::new(lsm_tree::compaction::PullDown(0, 6)), 0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + assert_eq!( + 0, + tree.current_version() + .level(6) + .expect("should exist") + .first() + .expect("should have at least 1 run") + .first() + .expect("should have one table") + .metadata + .tombstone_count + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_guarded_range.rs b/crates/lsm-tree/tests/tree_guarded_range.rs new file mode 100644 index 000000000..40d0623f3 --- /dev/null +++ b/crates/lsm-tree/tests/tree_guarded_range.rs @@ -0,0 +1,73 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_guarded_range() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 2); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 3); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 4); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 5); + + assert_eq!( + 2, + tree.range("a"..="f", SeqNo::MAX, None) + .flat_map(Guard::key) + .count(), + ); + assert_eq!( + 2, + tree.range("f"..="g", SeqNo::MAX, None) + .flat_map(Guard::key) + .count(), + ); + + Ok(()) +} + +#[test] +fn blob_tree_guarded_range() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 2); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 3); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 4); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 5); + + assert_eq!( + 2, + tree.range("a"..="f", SeqNo::MAX, None) + .flat_map(Guard::key) + .count(), + ); + assert_eq!( + 2, + tree.range("f"..="g", SeqNo::MAX, None) + .flat_map(Guard::key) + .count(), + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_kv.rs b/crates/lsm-tree/tests/tree_kv.rs new file mode 100644 index 000000000..af1d4d326 --- /dev/null +++ b/crates/lsm-tree/tests/tree_kv.rs @@ -0,0 +1,160 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_first_last_kv() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert!(tree.is_empty(SeqNo::MAX, None)?); + assert_eq!( + tree.first_key_value(SeqNo::MAX, None) + .map(|x| x.into_inner().unwrap()), + None + ); + assert_eq!( + tree.last_key_value(SeqNo::MAX, None) + .map(|x| x.into_inner().unwrap()), + None + ); + + tree.insert("b", "b", 0); + assert_eq!( + b"b", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"b", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + + tree.flush_active_memtable(0)?; + assert_eq!( + b"b", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"b", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + assert_eq!( + b"b", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"b", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + + tree.insert("a", "a", 0); + assert_eq!(2, tree.len(SeqNo::MAX, None)?); + assert_eq!( + b"a", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"b", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + + tree.insert("c", "c", 0); + assert_eq!(3, tree.len(SeqNo::MAX, None)?); + assert_eq!( + b"a", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"c", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + + tree.flush_active_memtable(0)?; + assert_eq!( + b"a", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"c", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(3, tree.len(SeqNo::MAX, None)?); + assert_eq!( + b"a", + &*tree + .first_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + assert_eq!( + b"c", + &*tree + .last_key_value(SeqNo::MAX, None) + .map(|x| x.key().unwrap()) + .unwrap() + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_l0_point_read.rs b/crates/lsm-tree/tests/tree_l0_point_read.rs new file mode 100644 index 000000000..230213a30 --- /dev/null +++ b/crates/lsm-tree/tests/tree_l0_point_read.rs @@ -0,0 +1,48 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_l0_point_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 1); + tree.insert("b", "b", 1); + tree.insert("c", "c", 1); + tree.flush_active_memtable(0)?; + + tree.insert("a", "A", 2); + tree.insert("b", "B", 2); + tree.insert("c", "C", 2); + tree.flush_active_memtable(0)?; + + tree.insert("d", "d", 3); + tree.insert("e", "e", 3); + tree.insert("f", "f", 3); + tree.flush_active_memtable(0)?; + + tree.insert("g", "g", 3); + tree.flush_active_memtable(0)?; + + assert_eq!(b"A", &*tree.get("a", SeqNo::MAX)?.unwrap()); + assert_eq!(b"B", &*tree.get("b", SeqNo::MAX)?.unwrap()); + assert_eq!(b"C", &*tree.get("c", SeqNo::MAX)?.unwrap()); + assert_eq!(b"d", &*tree.get("d", SeqNo::MAX)?.unwrap()); + assert_eq!(b"e", &*tree.get("e", SeqNo::MAX)?.unwrap()); + assert_eq!(b"f", &*tree.get("f", SeqNo::MAX)?.unwrap()); + assert_eq!(b"g", &*tree.get("g", SeqNo::MAX)?.unwrap()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_l0_range.rs b/crates/lsm-tree/tests/tree_l0_range.rs new file mode 100644 index 000000000..a8fbc0f12 --- /dev/null +++ b/crates/lsm-tree/tests/tree_l0_range.rs @@ -0,0 +1,51 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_l0_range_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = lsm_tree::Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 1); + tree.insert("b", "b", 1); + tree.insert("c", "c", 1); + tree.flush_active_memtable(0)?; + + tree.insert("a", "A", 2); + tree.insert("b", "B", 2); + tree.insert("c", "C", 2); + tree.flush_active_memtable(0)?; + + tree.insert("d", "d", 3); + tree.insert("e", "e", 3); + tree.insert("f", "f", 3); + tree.flush_active_memtable(0)?; + + tree.insert("g", "g".repeat(10_000), 3); + tree.flush_active_memtable(0)?; + + let mut range = tree.range("c"..="e", SeqNo::MAX, None); + assert_eq!(b"C", &*range.next().unwrap().value()?); + assert_eq!(b"d", &*range.next().unwrap().value()?); + assert_eq!(b"e", &*range.next().unwrap().value()?); + assert!(range.next().is_none()); + + let mut range = tree.range("f"..="g", SeqNo::MAX, None).rev(); + assert_eq!(b"g".repeat(10_000), &*range.next().unwrap().value()?); + assert_eq!(b"f", &*range.next().unwrap().value()?); + assert!(range.next().is_none()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_major_compaction.rs b/crates/lsm-tree/tests/tree_major_compaction.rs new file mode 100644 index 000000000..010efac19 --- /dev/null +++ b/crates/lsm-tree/tests/tree_major_compaction.rs @@ -0,0 +1,56 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_major_compaction() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(path, seqno.clone(), SequenceNumberCounter::default()).open()?; + + tree.insert("a".as_bytes(), "abc", seqno.next()); + tree.insert("b".as_bytes(), "abc", seqno.next()); + tree.insert("c".as_bytes(), "abc", seqno.next()); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + + tree.major_compact(u64::MAX, 1_000 /* NOTE: Simulate some time passing */)?; + assert_eq!(1, tree.table_count()); + + let item = tree.get_internal_entry(b"a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "a".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 0); + + let item = tree.get_internal_entry(b"b", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "b".as_bytes()); + assert!(!item.is_tombstone()); + // assert_eq!(item.key.seqno, 0); // NOTE: Seqno is zeroed because below GC threshold // TODO: + + let item = tree.get_internal_entry(b"c", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "c".as_bytes()); + assert!(!item.is_tombstone()); + // assert_eq!(item.key.seqno, 0); // NOTE: Seqno is zeroed because below GC threshold // TODO: + + assert_eq!(1, tree.table_count()); + assert_eq!(3, tree.len(SeqNo::MAX, None)?); + + let batch_seqno = seqno.next(); + tree.remove("a".as_bytes(), batch_seqno); + tree.remove("b".as_bytes(), batch_seqno); + tree.remove("c".as_bytes(), batch_seqno); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + + tree.major_compact(u64::MAX, 1_000 /* NOTE: Simulate some time passing */)?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_mvcc_simple.rs b/crates/lsm-tree/tests/tree_mvcc_simple.rs new file mode 100644 index 000000000..4d4853929 --- /dev/null +++ b/crates/lsm-tree/tests/tree_mvcc_simple.rs @@ -0,0 +1,115 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_read_mvcc() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a0", 0); + tree.insert("a", "a1", 1); + tree.insert("b", "b0", 0); + tree.insert("b", "b1", 1); + + tree.flush_active_memtable(0)?; + + tree.insert("b", "b2", 2); + tree.insert("b", "b3", 3); + tree.insert("c", "c4", 4); + + tree.flush_active_memtable(0)?; + + tree.insert("a", "a5", 5); + + assert_eq!(&*tree.get("a", SeqNo::MAX)?.unwrap(), b"a5"); + assert_eq!(&*tree.get("b", SeqNo::MAX)?.unwrap(), b"b3"); + assert_eq!(&*tree.get("c", SeqNo::MAX)?.unwrap(), b"c4"); + + assert_eq!(&*tree.get("a", 1)?.unwrap(), b"a0"); + assert_eq!(&*tree.get("b", 1)?.unwrap(), b"b0"); + assert!(tree.get("c", 1)?.is_none()); + + assert_eq!(&*tree.get("a", 2)?.unwrap(), b"a1"); + assert_eq!(&*tree.get("b", 2)?.unwrap(), b"b1"); + assert!(tree.get("c", 2)?.is_none()); + + assert_eq!(&*tree.get("a", 3)?.unwrap(), b"a1"); + assert_eq!(&*tree.get("b", 3)?.unwrap(), b"b2"); + assert!(tree.get("c", 3)?.is_none()); + + assert_eq!(&*tree.get("a", 4)?.unwrap(), b"a1"); + assert_eq!(&*tree.get("b", 4)?.unwrap(), b"b3"); + assert!(tree.get("c", 4)?.is_none()); + + assert_eq!(&*tree.get("a", 5)?.unwrap(), b"a1"); + assert_eq!(&*tree.get("b", 5)?.unwrap(), b"b3"); + assert_eq!(&*tree.get("c", 5)?.unwrap(), b"c4"); + + assert_eq!(&*tree.get("a", 6)?.unwrap(), b"a5"); + assert_eq!(&*tree.get("b", 6)?.unwrap(), b"b3"); + assert_eq!(&*tree.get("c", 6)?.unwrap(), b"c4"); + + assert_eq!(&*tree.get("a", 100)?.unwrap(), b"a5"); + assert_eq!(&*tree.get("b", 100)?.unwrap(), b"b3"); + assert_eq!(&*tree.get("c", 100)?.unwrap(), b"c4"); + + let mut iter = tree.iter(SeqNo::MAX, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a5"); + assert_eq!(&*iter.next().unwrap().value()?, b"b3"); + assert_eq!(&*iter.next().unwrap().value()?, b"c4"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(1, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a0"); + assert_eq!(&*iter.next().unwrap().value()?, b"b0"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(2, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a1"); + assert_eq!(&*iter.next().unwrap().value()?, b"b1"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(3, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a1"); + assert_eq!(&*iter.next().unwrap().value()?, b"b2"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(4, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a1"); + assert_eq!(&*iter.next().unwrap().value()?, b"b3"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(5, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a1"); + assert_eq!(&*iter.next().unwrap().value()?, b"b3"); + assert_eq!(&*iter.next().unwrap().value()?, b"c4"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(6, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a5"); + assert_eq!(&*iter.next().unwrap().value()?, b"b3"); + assert_eq!(&*iter.next().unwrap().value()?, b"c4"); + assert!(iter.next().is_none()); + + let mut iter = tree.iter(100, None); + + assert_eq!(&*iter.next().unwrap().value()?, b"a5"); + assert_eq!(&*iter.next().unwrap().value()?, b"b3"); + assert_eq!(&*iter.next().unwrap().value()?, b"c4"); + assert!(iter.next().is_none()); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_non_disjoint_point_read.rs b/crates/lsm-tree/tests/tree_non_disjoint_point_read.rs new file mode 100644 index 000000000..2cbbe8ccd --- /dev/null +++ b/crates/lsm-tree/tests/tree_non_disjoint_point_read.rs @@ -0,0 +1,61 @@ +use lsm_tree::{ + config::BlockSizePolicy, get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter, +}; +use test_log::test; + +#[test] +fn tree_non_disjoint_point_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .data_block_size_policy(BlockSizePolicy::all(1_024)) + // .index_block_size_policy(BlockSizePolicy::all(1_024)) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("c", "c", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 0); + tree.insert("d", "d", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 0); + tree.insert("e", "e", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 0); + tree.insert("f", "f", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 0); + tree.insert("g", "g", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 0); + tree.insert("h", "h", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.insert("a", "a", 0); + tree.insert("z", "z", 0); + tree.flush_active_memtable(0)?; + + tree.get("c", SeqNo::MAX).unwrap().unwrap(); + tree.get("d", SeqNo::MAX).unwrap().unwrap(); + tree.get("e", SeqNo::MAX).unwrap().unwrap(); + tree.get("f", SeqNo::MAX).unwrap().unwrap(); + tree.get("g", SeqNo::MAX).unwrap().unwrap(); + tree.get("h", SeqNo::MAX).unwrap().unwrap(); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_range.rs b/crates/lsm-tree/tests/tree_range.rs new file mode 100644 index 000000000..4d6ee2bbc --- /dev/null +++ b/crates/lsm-tree/tests/tree_range.rs @@ -0,0 +1,133 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_range_count() -> lsm_tree::Result<()> { + use std::ops::Bound::{self, Excluded, Unbounded}; + + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 2); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 3); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 4); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 5); + + assert_eq!(2, tree.range("a"..="f", SeqNo::MAX, None).count()); + assert_eq!(2, tree.range("f"..="g", SeqNo::MAX, None).count()); + + assert_eq!( + 1, + tree.range::, (Bound>, Bound>)>( + (Excluded("f".into()), Unbounded), + SeqNo::MAX, + None + ) + .count() + ); + + tree.flush_active_memtable(0)?; + + assert_eq!(2, tree.range("a"..="f", SeqNo::MAX, None).count()); + assert_eq!( + 1, + tree.range::, (Bound>, Bound>)>( + (Excluded("f".into()), Unbounded), + SeqNo::MAX, + None + ) + .count() + ); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 6); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 7); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 8); + + assert_eq!(2, tree.range("a"..="f", SeqNo::MAX, None).count()); + assert_eq!( + 1, + tree.range::, (Bound>, Bound>)>( + (Excluded("f".into()), Unbounded), + SeqNo::MAX, + None + ) + .count() + ); + + Ok(()) +} + +#[test] +fn blob_tree_range_count() -> lsm_tree::Result<()> { + use std::ops::Bound::{self, Excluded, Unbounded}; + + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("g".as_bytes(), b"neptune!".repeat(128_000), 2); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 3); + tree.insert("f".as_bytes(), b"neptune!".repeat(128_000), 4); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 5); + + assert_eq!(2, tree.range("a"..="f", SeqNo::MAX, None).count()); + assert_eq!(2, tree.range("f"..="g", SeqNo::MAX, None).count()); + + assert_eq!( + 1, + tree.range::, (Bound>, Bound>)>( + (Excluded("f".into()), Unbounded), + SeqNo::MAX, + None + ) + .count() + ); + + tree.flush_active_memtable(0)?; + + assert_eq!(2, tree.range("a"..="f", SeqNo::MAX, None).count()); + assert_eq!( + 1, + tree.range::, (Bound>, Bound>)>( + (Excluded("f".into()), Unbounded), + SeqNo::MAX, + None + ) + .count() + ); + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 6); + tree.insert("f".as_bytes(), nanoid::nanoid!().as_bytes(), 7); + tree.insert("g".as_bytes(), nanoid::nanoid!().as_bytes(), 8); + + assert_eq!(2, tree.range("a"..="f", SeqNo::MAX, None).count()); + assert_eq!( + 1, + tree.range::, (Bound>, Bound>)>( + (Excluded("f".into()), Unbounded), + SeqNo::MAX, + None + ) + .count() + ); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_range_memtable_only.rs b/crates/lsm-tree/tests/tree_range_memtable_only.rs new file mode 100644 index 000000000..19038d7e9 --- /dev/null +++ b/crates/lsm-tree/tests/tree_range_memtable_only.rs @@ -0,0 +1,80 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_range_memtable_only() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "", 0); + tree.insert("b", "", 0); + tree.insert("c", "", 0); + + let found: Vec = tree + .range("a".."a", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(Vec::::new(), found); + + let found = tree + .range("a"..="a", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(vec!["a"], found); + + let found = tree + .range("a".."b", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(vec!["a"], found); + + let found = tree + .range("a"..="b", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(vec!["a", "b"], found); + + let found = tree + .range("a".."a", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .rev() + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(Vec::::new(), found); + + let found = tree + .range("a"..="a", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .rev() + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(vec!["a"], found); + + let found = tree + .range("a".."b", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .rev() + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(vec!["a"], found); + + let found = tree + .range("a"..="b", SeqNo::MAX, None) + .flat_map(|x| x.key()) + .rev() + .map(|k| String::from_utf8(k.to_vec()).unwrap()) + .collect::>(); + assert_eq!(vec!["b", "a"], found); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_recover_counter.rs b/crates/lsm-tree/tests/tree_recover_counter.rs new file mode 100644 index 000000000..e8474c627 --- /dev/null +++ b/crates/lsm-tree/tests/tree_recover_counter.rs @@ -0,0 +1,36 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_recover_table_counter() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let counter_expected = { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + + tree.next_table_id() + }; + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(counter_expected, tree.next_table_id()); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_recover_large_value.rs b/crates/lsm-tree/tests/tree_recover_large_value.rs new file mode 100644 index 000000000..102d08dc5 --- /dev/null +++ b/crates/lsm-tree/tests/tree_recover_large_value.rs @@ -0,0 +1,33 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_recover_large_value() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + tree.insert("a", "a".repeat(100_000), 0); + tree.flush_active_memtable(0)?; + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!( + &*tree.get("a", SeqNo::MAX)?.expect("should exist"), + "a".repeat(100_000).as_bytes() + ); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_recovery_versions.rs b/crates/lsm-tree/tests/tree_recovery_versions.rs new file mode 100644 index 000000000..b832f5554 --- /dev/null +++ b/crates/lsm-tree/tests/tree_recovery_versions.rs @@ -0,0 +1,48 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +#[ignore = "restore Version history maintenance"] +fn tree_recovery_version_free_list() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + { + let tree = Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert!(path.join("v0").try_exists()?); + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.version_free_list_len()); + assert!(path.join("v1").try_exists()?); + + tree.insert("b", "b", 0); + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.version_free_list_len()); + assert!(path.join("v2").try_exists()?); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(0, tree.version_free_list_len()); + assert!(!path.join("v0").try_exists()?); + assert!(!path.join("v1").try_exists()?); + assert!(path.join("v2").try_exists()?); + + assert!(tree.contains_key("a", 1)?); + assert!(tree.contains_key("b", 1)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_reload.rs b/crates/lsm-tree/tests/tree_reload.rs new file mode 100644 index 000000000..0d396388a --- /dev/null +++ b/crates/lsm-tree/tests/tree_reload.rs @@ -0,0 +1,264 @@ +use lsm_tree::{ + get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter, TreeType, +}; +use test_log::test; + +const ITEM_COUNT: usize = 10_000; + +#[test] +fn tree_reload_smoke_test() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(0, tree.table_count()); + + tree.insert("a", "a", 0); + tree.flush_active_memtable(0)?; + + assert_eq!(1, tree.table_count()); + assert!(tree.contains_key("a", SeqNo::MAX)?); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(1, tree.table_count()); + assert!(tree.contains_key("a", SeqNo::MAX)?); + } + + Ok(()) +} + +#[test] +fn tree_reload_smoke_test_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let large_value = "a".repeat(10_000); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(0, tree.table_count()); + + tree.insert("a", &large_value, 0); + tree.flush_active_memtable(0)?; + + assert_eq!(1, tree.table_count()); + assert!(tree.contains_key("a", SeqNo::MAX)?); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(1, tree.table_count()); + assert_eq!(large_value.as_bytes(), tree.get("a", SeqNo::MAX)?.unwrap()); + } + + Ok(()) +} + +#[test] +fn tree_reload_blob_again_without_opts() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + assert_eq!(0, tree.table_count()); + assert_eq!(0, tree.blob_file_count()); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(None) + .open(); + + assert!(matches!(tree, Err(lsm_tree::Error::Unrecoverable))); + } + + Ok(()) +} + +#[test] +fn tree_reload_empty() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Standard); + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Standard); + + tree.flush_active_memtable(0)?; + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Standard); + + tree.flush_active_memtable(0)?; + } + + { + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert_eq!(tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), 0); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + 0 + ); + assert_eq!(tree.tree_type(), TreeType::Standard); + } + + Ok(()) +} + +#[test] +fn tree_reload() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + { + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), seqno.next()); + } + + tree.flush_active_memtable(0)?; + + for x in 0..ITEM_COUNT as u64 { + let key: [u8; 8] = (x + ITEM_COUNT as u64).to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT * 2 + ); + + tree.flush_active_memtable(0)?; + } + + { + let tree = Config::new(&folder, seqno, SequenceNumberCounter::default()).open()?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.iter(SeqNo::MAX, None).flat_map(|x| x.key()).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.iter(SeqNo::MAX, None) + .rev() + .flat_map(|x| x.key()) + .count(), + ITEM_COUNT * 2 + ); + } + + std::thread::sleep(std::time::Duration::from_secs(2)); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_reload_pwd.rs b/crates/lsm-tree/tests/tree_reload_pwd.rs new file mode 100644 index 000000000..ca0e65805 --- /dev/null +++ b/crates/lsm-tree/tests/tree_reload_pwd.rs @@ -0,0 +1,50 @@ +use fs_extra::dir::CopyOptions; +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +const ITEM_COUNT: usize = 10_000; + +#[test] +fn tree_reload_pwd() -> lsm_tree::Result<()> { + let folder_old = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + { + let tree = + Config::new(&folder_old, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = nanoid::nanoid!(); + tree.insert(key, value.as_bytes(), seqno.next()); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(ITEM_COUNT, tree.len(SeqNo::MAX, None)?); + } + + let folder_new = tempfile::tempdir()?; + let folder_new_subfolder = folder_new.path().join("deep"); + std::fs::create_dir_all(&folder_new_subfolder)?; + + fs_extra::dir::move_dir( + folder_old, + &folder_new_subfolder, + &CopyOptions::default().content_only(true).overwrite(true), + ) + .expect("should move"); + + { + let tree = Config::new( + &folder_new_subfolder, + seqno, + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(ITEM_COUNT, tree.len(SeqNo::MAX, None)?); + } + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_sealed_shadowing.rs b/crates/lsm-tree/tests/tree_sealed_shadowing.rs new file mode 100644 index 000000000..d61eed689 --- /dev/null +++ b/crates/lsm-tree/tests/tree_sealed_shadowing.rs @@ -0,0 +1,41 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_sealed_memtable_tombstone_shadowing() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "123", 0); + assert!(tree.contains_key("a", SeqNo::MAX)?); + + tree.flush_active_memtable(0)?; + + tree.remove("a", 1); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + tree.rotate_memtable().unwrap(); + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + { + let flush_lock = tree.get_flush_lock(); + assert!(tree.flush(&flush_lock, 0)?.unwrap() > 0); + } + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + tree.major_compact(u64::MAX, 2)?; + + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_seqno.rs b/crates/lsm-tree/tests/tree_seqno.rs new file mode 100644 index 000000000..b8b0752af --- /dev/null +++ b/crates/lsm-tree/tests/tree_seqno.rs @@ -0,0 +1,64 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_highest_seqno() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + assert_eq!(tree.get_highest_seqno(), None); + assert_eq!(tree.get_highest_memtable_seqno(), None); + assert_eq!(tree.get_highest_persisted_seqno(), None); + + tree.insert("a", "a0", 0); + assert_eq!(tree.get_highest_seqno(), Some(0)); + assert_eq!(tree.get_highest_memtable_seqno(), Some(0)); + assert_eq!(tree.get_highest_persisted_seqno(), None); + + tree.insert("a", "a1", 1); + assert_eq!(tree.get_highest_seqno(), Some(1)); + assert_eq!(tree.get_highest_memtable_seqno(), Some(1)); + assert_eq!(tree.get_highest_persisted_seqno(), None); + + tree.insert("b", "b0", 2); + assert_eq!(tree.get_highest_seqno(), Some(2)); + assert_eq!(tree.get_highest_memtable_seqno(), Some(2)); + assert_eq!(tree.get_highest_persisted_seqno(), None); + + tree.insert("b", "b1", 3); + assert_eq!(tree.get_highest_seqno(), Some(3)); + assert_eq!(tree.get_highest_memtable_seqno(), Some(3)); + assert_eq!(tree.get_highest_persisted_seqno(), None); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.get_highest_seqno(), Some(3)); + assert_eq!(tree.get_highest_memtable_seqno(), None); + assert_eq!(tree.get_highest_persisted_seqno(), Some(3)); + + tree.insert("a", "a0", 4); + assert_eq!(tree.get_highest_seqno(), Some(4)); + assert_eq!(tree.get_highest_memtable_seqno(), Some(4)); + assert_eq!(tree.get_highest_persisted_seqno(), Some(3)); + + tree.rotate_memtable().unwrap(); + + assert_eq!(tree.get_highest_seqno(), Some(4)); + assert_eq!(tree.get_highest_memtable_seqno(), Some(4)); + assert_eq!(tree.get_highest_persisted_seqno(), Some(3)); + + { + let flush_lock = tree.get_flush_lock(); + assert!(tree.flush(&flush_lock, 0)?.unwrap() > 0); + } + + assert_eq!(tree.get_highest_seqno(), Some(4)); + assert_eq!(tree.get_highest_memtable_seqno(), None); + assert_eq!(tree.get_highest_persisted_seqno(), Some(4)); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_shadowing.rs b/crates/lsm-tree/tests/tree_shadowing.rs new file mode 100644 index 000000000..710bb6a28 --- /dev/null +++ b/crates/lsm-tree/tests/tree_shadowing.rs @@ -0,0 +1,381 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_shadowing_upsert() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let key = "1".as_bytes(); + let value = "oldvalue".as_bytes(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + tree.insert(key, value, 0); + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + let value = "newvalue".as_bytes(); + + tree.insert(key, value, 1); + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + Ok(()) +} + +#[test] +fn tree_shadowing_upsert_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + let key = "1".as_bytes(); + let value = "oldvalue".as_bytes(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + tree.insert(key, value, 0); + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + let value = "newvalue".as_bytes(); + + tree.insert(key, value, 1); + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + Ok(()) +} + +#[test] +fn tree_shadowing_delete() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open() + .unwrap(); + + let key = "1".as_bytes(); + let value = "oldvalue".as_bytes(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + tree.insert(key, value, 0); + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.remove(key, 1); + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.get(key, SeqNo::MAX)?.is_none()); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.get(key, SeqNo::MAX)?.is_none()); + + Ok(()) +} + +#[test] +fn tree_shadowing_delete_blob() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .with_kv_separation(Some(Default::default())) + .open()?; + + let key = "1".as_bytes(); + let value = "oldvalue".as_bytes(); + + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + tree.insert(key, value, 0); + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 1); + assert_eq!(tree.get(key, SeqNo::MAX)?, Some(value.into())); + + tree.remove(key, 1); + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.get(key, SeqNo::MAX)?.is_none()); + + tree.flush_active_memtable(0)?; + assert_eq!(tree.len(SeqNo::MAX, None)?, 0); + assert!(tree.get(key, SeqNo::MAX)?.is_none()); + + Ok(()) +} + +#[test] +fn tree_shadowing_range() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 10_000; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = "old".as_bytes(); + tree.insert(key, value, seqno.next()); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "old".as_bytes())); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = "new".as_bytes(); + tree.insert(key, value, seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + Ok(()) +} + +#[test] +fn tree_shadowing_range_blob() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 10_000; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .with_kv_separation(Some(Default::default())) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = "old".as_bytes(); + tree.insert(key, value, seqno.next()); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "old".as_bytes())); + + for x in 0..ITEM_COUNT as u64 { + let key = x.to_be_bytes(); + let value = "new".as_bytes(); + tree.insert(key, value, seqno.next()); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + Ok(()) +} + +#[test] +fn tree_shadowing_prefix() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 10_000; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()).open()?; + + for x in 0..ITEM_COUNT as u64 { + let value = "old".as_bytes(); + let batch_seqno = seqno.next(); + + tree.insert(format!("pre:{x}").as_bytes(), value, batch_seqno); + tree.insert(format!("prefix:{x}").as_bytes(), value, batch_seqno); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("pre".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.prefix("prefix".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "old".as_bytes())); + + for x in 0..ITEM_COUNT as u64 { + let value = "new".as_bytes(); + let batch_seqno = seqno.next(); + + tree.insert(format!("pre:{x}").as_bytes(), value, batch_seqno); + tree.insert(format!("prefix:{x}").as_bytes(), value, batch_seqno); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("pre".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.prefix("prefix".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("pre".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.prefix("prefix".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + Ok(()) +} + +#[test] +fn tree_shadowing_prefix_blob() -> lsm_tree::Result<()> { + const ITEM_COUNT: usize = 10_000; + + let folder = get_tmp_folder(); + + let seqno = SequenceNumberCounter::default(); + + let tree = Config::new(&folder, seqno.clone(), SequenceNumberCounter::default()) + .with_kv_separation(Some(Default::default())) + .open()?; + + for x in 0..ITEM_COUNT as u64 { + let value = "old".as_bytes(); + let batch_seqno = seqno.next(); + + tree.insert(format!("pre:{x}").as_bytes(), value, batch_seqno); + tree.insert(format!("prefix:{x}").as_bytes(), value, batch_seqno); + } + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("pre".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.prefix("prefix".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "old".as_bytes())); + + for x in 0..ITEM_COUNT as u64 { + let value = "new".as_bytes(); + let batch_seqno = seqno.next(); + + tree.insert(format!("pre:{x}").as_bytes(), value, batch_seqno); + tree.insert(format!("prefix:{x}").as_bytes(), value, batch_seqno); + } + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("pre".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.prefix("prefix".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + tree.flush_active_memtable(0)?; + + assert_eq!(tree.len(SeqNo::MAX, None)?, ITEM_COUNT * 2); + assert_eq!( + tree.prefix("pre".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT * 2 + ); + assert_eq!( + tree.prefix("prefix".as_bytes(), SeqNo::MAX, None).count(), + ITEM_COUNT + ); + assert!(tree + .iter(SeqNo::MAX, None) + .all(|x| &*x.value().unwrap() == "new".as_bytes())); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_trace.rs b/crates/lsm-tree/tests/tree_trace.rs new file mode 100644 index 000000000..3bdac9dc0 --- /dev/null +++ b/crates/lsm-tree/tests/tree_trace.rs @@ -0,0 +1,34 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_trace() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let key = "1".as_bytes(); + + let value = "oldvalue".as_bytes(); + tree.insert(key, value, 0); + + let value = "newvalue".as_bytes(); + tree.insert(key, value, 1); + + tree.flush_active_memtable(0)?; + + let value = "dsadsa".as_bytes(); + tree.insert(key, value, 2); + tree.rotate_memtable(); + + let value = "yxcyxc".as_bytes(); + tree.insert(key, value, 3); + tree.print_trace(key)?; + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_v1_load_fixture.rs b/crates/lsm-tree/tests/tree_v1_load_fixture.rs new file mode 100644 index 000000000..ee968bc7c --- /dev/null +++ b/crates/lsm-tree/tests/tree_v1_load_fixture.rs @@ -0,0 +1,34 @@ +use lsm_tree::{Config, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_load_v1() -> lsm_tree::Result<()> { + let folder = "test_fixture/v1_tree"; + + let result = Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open(); + + matches!(result, Err(lsm_tree::Error::InvalidVersion(1))); + + Ok(()) +} + +#[test] +fn tree_load_v1_corrupt() -> lsm_tree::Result<()> { + let folder = "test_fixture/v1_tree_corrupt"; + + let result = Config::new( + folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open(); + + matches!(result, Err(lsm_tree::Error::InvalidVersion(1))); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_weak_delete.rs b/crates/lsm-tree/tests/tree_weak_delete.rs new file mode 100644 index 000000000..68fc306a6 --- /dev/null +++ b/crates/lsm-tree/tests/tree_weak_delete.rs @@ -0,0 +1,167 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_weak_delete_simple() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "old", 0); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + assert!(tree.contains_key("a", SeqNo::MAX)?); + + tree.remove_weak("a", 1); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + tree.insert("a", "new", 2); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + assert!(tree.contains_key("a", SeqNo::MAX)?); + + tree.remove_weak("a", 3); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_weak_delete_flush() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "old", 0); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + + tree.remove_weak("a", 1); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn tree_weak_delete_semi_flush() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "old", 0); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + + tree.remove_weak("a", 1); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn tree_weak_delete_flush_point_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "old", 0); + assert!(tree.contains_key("a", SeqNo::MAX)?); + + tree.remove_weak("a", 1); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_weak_delete_semi_flush_point_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "old", 0); + assert!(tree.contains_key("a", SeqNo::MAX)?); + tree.flush_active_memtable(0)?; + assert_eq!(1, tree.table_count()); + + tree.remove_weak("a", 1); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + tree.flush_active_memtable(0)?; + assert_eq!(2, tree.table_count()); + assert!(!tree.contains_key("a", SeqNo::MAX)?); + + Ok(()) +} + +#[test] +fn tree_weak_delete_resurrection() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "old", 0); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + + tree.insert("a", "new", 1); + assert_eq!(1, tree.len(SeqNo::MAX, None)?); + + tree.remove_weak("a", 2); + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_weak_delete_eviction.rs b/crates/lsm-tree/tests/tree_weak_delete_eviction.rs new file mode 100644 index 000000000..d36dc09fc --- /dev/null +++ b/crates/lsm-tree/tests/tree_weak_delete_eviction.rs @@ -0,0 +1,30 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_weak_remove_flush_eviction() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + for (idx, c) in ('a'..='z').map(|x| (x as u8).to_be_bytes()).enumerate() { + tree.insert(c, c, idx as SeqNo); + } + + for (idx, c) in ('a'..='z').map(|x| (x as u8).to_be_bytes()).enumerate() { + tree.remove_weak(c, idx as SeqNo + 1000); + } + + assert_eq!(0, tree.len(SeqNo::MAX, None)?); + + tree.flush_active_memtable(1_100)?; + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_weak_delete_queue.rs b/crates/lsm-tree/tests/tree_weak_delete_queue.rs new file mode 100644 index 000000000..b03d9d5b4 --- /dev/null +++ b/crates/lsm-tree/tests/tree_weak_delete_queue.rs @@ -0,0 +1,107 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Guard, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_weak_delete_queue() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.insert("d", "d", 0); + tree.insert("e", "e", 0); + assert_eq!( + b"a", + &*tree.first_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("a", 1); + assert_eq!( + b"b", + &*tree.first_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("b", 1); + assert_eq!( + b"c", + &*tree.first_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("c", 1); + assert_eq!( + b"d", + &*tree.first_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("d", 1); + assert_eq!( + b"e", + &*tree.first_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("e", 1); + assert!(tree.is_empty(SeqNo::MAX, None)?); + + Ok(()) +} + +#[test] +fn tree_weak_delete_queue_reverse() -> lsm_tree::Result<()> { + let folder = tempfile::tempdir()?; + let path = folder.path(); + + let tree = lsm_tree::Config::new( + path, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a", "a", 0); + tree.insert("b", "b", 0); + tree.insert("c", "c", 0); + tree.insert("d", "d", 0); + tree.insert("e", "e", 0); + assert_eq!( + b"e", + &*tree.last_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("e", 1); + assert_eq!( + b"d", + &*tree.last_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("d", 1); + assert_eq!( + b"c", + &*tree.last_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("c", 1); + assert_eq!( + b"b", + &*tree.last_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("b", 1); + assert_eq!( + b"a", + &*tree.last_key_value(SeqNo::MAX, None).unwrap().key()? + ); + + tree.remove_weak("a", 1); + assert!(tree.is_empty(SeqNo::MAX, None)?); + + Ok(()) +} diff --git a/crates/lsm-tree/tests/tree_write_read.rs b/crates/lsm-tree/tests/tree_write_read.rs new file mode 100644 index 000000000..af4e3508d --- /dev/null +++ b/crates/lsm-tree/tests/tree_write_read.rs @@ -0,0 +1,59 @@ +use lsm_tree::{get_tmp_folder, AbstractTree, Config, SeqNo, SequenceNumberCounter}; +use test_log::test; + +#[test] +fn tree_write_and_read() -> lsm_tree::Result<()> { + let folder = get_tmp_folder(); + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + tree.insert("a".as_bytes(), nanoid::nanoid!().as_bytes(), 0); + tree.insert("b".as_bytes(), nanoid::nanoid!().as_bytes(), 1); + tree.insert("c".as_bytes(), nanoid::nanoid!().as_bytes(), 2); + + let item = tree.get_internal_entry(b"a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "a".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 0); + + let item = tree.get_internal_entry(b"b", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "b".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 1); + + let item = tree.get_internal_entry(b"c", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "c".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 2); + + tree.flush_active_memtable(0)?; + + let tree = Config::new( + &folder, + SequenceNumberCounter::default(), + SequenceNumberCounter::default(), + ) + .open()?; + + let item = tree.get_internal_entry(b"a", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "a".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 0); + + let item = tree.get_internal_entry(b"b", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "b".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 1); + + let item = tree.get_internal_entry(b"c", SeqNo::MAX)?.unwrap(); + assert_eq!(&*item.key.user_key, "c".as_bytes()); + assert!(!item.is_tombstone()); + assert_eq!(item.key.seqno, 2); + + Ok(()) +} diff --git a/crates/quickmatch/.gitignore b/crates/quickmatch/.gitignore new file mode 100644 index 000000000..5b93c2be3 --- /dev/null +++ b/crates/quickmatch/.gitignore @@ -0,0 +1,7 @@ +# Build +/target + +# Apple +.DS_Store + +*metrics* diff --git a/crates/quickmatch/Cargo.toml b/crates/quickmatch/Cargo.toml new file mode 100644 index 000000000..27b1ec97d --- /dev/null +++ b/crates/quickmatch/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "quickmatch" +description = "Lightning-fast fuzzy string matching with hybrid word and trigram indexing" +version.workspace = true +readme.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +homepage.workspace = true +keywords = ["search", "fuzzy", "matching", "trigram", "autocomplete"] +categories = ["algorithms", "text-processing", "data-structures"] + +[dependencies] +rustc-hash = { workspace = true } diff --git a/crates/quickmatch/docs/CHANGELOG.md b/crates/quickmatch/docs/CHANGELOG.md new file mode 100644 index 000000000..b47dd5fd7 --- /dev/null +++ b/crates/quickmatch/docs/CHANGELOG.md @@ -0,0 +1,161 @@ + + +# Changelog + +## [v0.4.0](https://github.com/nym21/quickmatch/releases/tag/v0.4.0) - 2026-03-22 + +### Breaking Changes + +#### Rust & JavaScript +- Word indexing now stores all prefixes of each word and compound, increasing memory usage proportionally to total word character count — previously only full words and full compounds were indexed ([Rust source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/lib.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/index.js)) + +### New Features + +#### Rust & JavaScript +- Added prefix indexing: every prefix of each word is now stored in the word index (e.g., "dominance" registers "d", "do", "dom", ..., "dominance"), so short queries like "dom" directly match items containing "dominance" without relying on trigram fallback ([Rust source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/lib.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/index.js)) +- Added prefix indexing for compound words: adjacent word pairs index all prefixes starting from the second word's first character (e.g., "hash" + "rate" indexes "hashr", "hashra", ..., "hashrate"), enabling partial compound matching +- Changed `prefix_score` from exact word matching to prefix matching (`starts_with` in Rust, length comparison in JS), so query word "pric" is recognized as a prefix of item word "price" and scored accordingly ([Rust source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/lib.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/index.js)) +- Added union fallback: when word intersection yields no common items and trigram matching produces no results, the algorithm falls back to union of all known word sets, returning partial matches instead of empty results ([Rust source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/lib.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/index.js)) +- Changed search flow to try trigram matching first when unknown words exist; if trigram results are found they're returned immediately, otherwise falls back to ranking known-word candidates (intersection or union) + +#### Examples +- Added `examples/compare.rs` with 80+ test queries covering exact matches, compound words, typos, short prefixes, acronyms, reverse word order, and unknown terms against a metrics dataset ([source](https://github.com/nym21/quickmatch/blob/v0.4.0/examples/compare.rs)) + +### Internal Changes + +#### Rust +- Extracted ranking logic into a standalone `fn rank()` method that takes an iterator of `(*const str, usize)` candidates and produces bucket-sorted results ([source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/lib.rs)) +- Added `intersect_sets()` method that returns `Option` (returning `None` instead of empty set to distinguish "no known words" from "empty intersection") +- Added `union_sets()` method that merges multiple `FxHashSet` references into a single set +- Changed query word deduplication from a separate `FxHashSet` to inline `seen.insert()` filtering during collection, then dropping the set immediately +- Used lifetime elision (`QuickMatch<'_>`) in `Send` and `Sync` trait implementations +- Removed doc comments from `prefix_score` free function + +#### JavaScript +- Replaced flat sorting by `(prefixScore, trigramScore, length)` with a 3-bucket ranking approach: items are bucketed by prefix score (0, 1, 2), each bucket is independently sorted by `(score, length)`, and results are filled from the highest bucket first until the limit is reached ([source](https://github.com/nym21/quickmatch/blob/v0.4.0/src/index.js)) +- Added `union()` helper function that merges multiple index arrays into a deduplicated result using a `Set` +- Changed `intersect()` to return `null` instead of an empty array when intersection is empty, distinguishing "no results" from "no input" +- Added duplicate prevention in `addToIndex()`: checks `arr[arr.length - 1] !== value` before pushing, preventing duplicate entries from prefix indexing +- Removed standalone `indexTrigrams()` function; trigram indexing is now inlined in the constructor alongside prefix indexing +- Condensed JSDoc class-level documentation to include behavioral descriptions (e.g., "Supports exact words, prefixes, joined words, and typo tolerance") + +[View changes](https://github.com/nym21/quickmatch/compare/v0.3.2...v0.4.0) + +--- + +## [v0.3.2](https://github.com/nym21/quickmatch/releases/tag/v0.3.2) - 2026-03-21 + +### Breaking Changes + +#### Rust & JavaScript +- Default separators changed from `['_', '-', ' ']` to `['_', '-', ' ', ':', '/']`, meaning colon-separated and path-like strings are now split into words by default ([Rust source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/config.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/index.js)) + +### New Features + +#### Rust & JavaScript +- Added compound word indexing: adjacent words in items are indexed as concatenated pairs (e.g., "hash" + "rate" → "hashrate"), so queries like `hashrate` now match items like `hash_rate` without relying on trigram fallback ([Rust source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/lib.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/index.js)) +- Added prefix-based result ranking via a new `prefix_score` function that scores items as exact match (2), prefix match (1), or no match (0) — results are now sorted by prefix score first, then trigram score, then length, producing significantly more relevant ordering for autocomplete ([Rust source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/lib.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/index.js)) +- Added `with_min_score(n)` (Rust) / `withMinScore(n)` (JS) configuration option to set the minimum trigram score required for fuzzy matches — higher values require more trigram overlap, reducing noise in results (default: 2) ([Rust source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/config.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/index.js)) + +#### Documentation +- Added `docs/README.md` with comprehensive usage examples for both Rust and JavaScript, a "How it works" section explaining the 3-stage matching pipeline (word → compound → trigram), a configuration options table, and performance benchmarks (~26μs/query Rust, ~29μs/query JS) ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/docs/README.md)) + +#### Examples +- Added `examples/colon_test.rs` and `examples/colon_test.mjs` demonstrating separator behavior with colon-containing items and queries under both default and custom separator configurations ([Rust source](https://github.com/nym21/quickmatch/blob/v0.3.2/examples/colon_test.rs), [JS source](https://github.com/nym21/quickmatch/blob/v0.3.2/examples/colon_test.mjs)) + +#### CI +- Added `.github/workflows/outdated.yml` workflow that runs daily (and on manual trigger) to detect outdated Rust dependencies using `cargo-outdated` ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/.github/workflows/outdated.yml)) + +### Bug Fixes + +#### Rust +- Fixed `max_word_len` calculation that was incorrectly using the full item length instead of individual word length, which could cause valid long words to be rejected from queries ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/lib.rs)) +- Fixed query normalization ordering so that emptiness and length checks now happen after trimming whitespace and lowercasing, correctly rejecting whitespace-only or non-ASCII-only queries ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/lib.rs)) + +### Internal Changes + +#### Rust +- Changed `query_words` collection from `FxHashSet` to `Vec` to preserve word ordering, which is required for the new prefix scoring algorithm ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/lib.rs)) +- Removed doc comments from `matches()` and `matches_with()` public methods + +#### JavaScript +- Replaced string-based separator checking (`separators.includes()`) with a `Uint8Array(128)` lookup table for O(1) separator detection instead of O(n) string scanning ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/index.js)) +- Replaced `Map` score tracking with a pre-allocated `Uint32Array(items.length)` and a dirty index list, eliminating Map allocation overhead during each search ([source](https://github.com/nym21/quickmatch/blob/v0.3.2/src/index.js)) +- Merged `sortedByLength()` and `rankedResults()` into a single `_rank()` method that handles both exact-match and fuzzy-match result paths +- Renamed internal functions for brevity: `normalizeQuery` → `normalize`, `parseWords` → `splitWords`, `addTrigramsToIndex` → `indexTrigrams`, `intersectAll` → `intersect`, `binarySearch` → `bsearch`, `pickTrigramPosition` → `trigramPosition`, `scoreByTrigrams` → `_scoreTrigrams` +- Condensed JSDoc annotations to single-line `@param` format + +#### Workspace +- Reorganized project files: moved `LICENSE` to `docs/LICENSE.md`, set `Cargo.toml` readme field to `docs/README.md` +- Added `*metrics*` pattern to `.gitignore` +- Added `examples` to `jsconfig.json` exclude list + +[View changes](https://github.com/nym21/quickmatch/compare/v0.3.1...v0.3.2) + +--- + +## [v0.3.1](https://github.com/nym21/quickmatch/releases/tag/v0.3.1) - 2026-01-22 + +### New Features + +#### JavaScript +- Added comprehensive JSDoc type annotations for `QuickMatchConfig` class properties (`separators`, `limit`, `trigramBudget`) and all public methods (`withLimit()`, `withTrigramBudget()`, `withSeparators()`, `matches()`, `matchesWith()`) ([source](https://github.com/nym21/quickmatch/blob/v0.3.1/src/index.js)) +- Added `@private` annotations for internal methods (`scoreByTrigrams()`, `sortedByLength()`, `rankedResults()`) to indicate non-public API +- Added explicit type annotations to Map types: `Map` for `wordIndex` and `trigramIndex` +- Added `jsconfig.json` with TypeScript-style type checking enabled (`checkJs: true`, `strict: true`) for improved IDE support and static analysis ([source](https://github.com/nym21/quickmatch/blob/v0.3.1/jsconfig.json)) + +### Internal Changes + +#### JavaScript +- Renamed `trigramCount` variable to `hitCount` for consistency with Rust implementation +- Normalized string quotes to double quotes throughout the codebase +- Reformatted multi-line function parameters for readability + +[View changes](https://github.com/nym21/quickmatch/compare/v0.3.0...v0.3.1) + +--- + +## [v0.3.0](https://github.com/nym21/quickmatch/releases/tag/v0.3.0) - 2026-01-22 + +### New Features + +#### JavaScript +- Added complete JavaScript port of the library with identical API to Rust, published as `quickmatch-js` npm package ([source](https://github.com/nym21/quickmatch/blob/v0.3.0/src/index.js)) +- Implemented `QuickMatch` class with word indexing and trigram-based fuzzy matching +- Implemented `QuickMatchConfig` class with builder methods (`withLimit()`, `withTrigramBudget()`, `withSeparators()`) +- Implemented `matches()` and `matchesWith()` methods returning results sorted by relevance +- Uses ES modules (`"type": "module"`) for modern JavaScript compatibility + +#### Documentation +- Added JavaScript installation instructions to README (`npm install quickmatch-js`) + +### Internal Changes + +#### Rust +- Renamed `trigram_count` to `hit_count` to clarify it tracks successful trigram matches rather than total attempts ([source](https://github.com/nym21/quickmatch/blob/v0.3.0/src/lib.rs)) +- Changed trigram budget tracking from increment-based (`trigram_count += 1`) to decrement-based (`budget -= 1`) for clearer remaining budget visibility +- Simplified trigram processing loop by removing `processed_trigrams` flag and using direct budget check (`if budget == 0`) for early termination + +[View changes](https://github.com/nym21/quickmatch/compare/v0.2.0...v0.3.0) + +--- + +## [v0.2.0](https://github.com/nym21/quickmatch/releases/tag/v0.2.0) - 2026-01-14 + +### New Features + +#### Rust +- Initial release of `quickmatch` - a fuzzy string matching library for autocomplete and search-as-you-type interfaces +- Added `QuickMatch` struct implementing hybrid matching with word-level indexing and trigram-based fuzzy fallback ([source](https://github.com/nym21/quickmatch/blob/v0.2.0/src/lib.rs)) +- Added `QuickMatchConfig` for customizing match behavior with builder pattern methods ([source](https://github.com/nym21/quickmatch/blob/v0.2.0/src/config.rs)): + - `with_limit(n)` - Set maximum results (default: 100) + - `with_trigram_budget(n)` - Set fuzzy matching depth (default: 6, range: 0-20) + - `with_separators(chars)` - Set word separator characters (default: `_`, `-`, ` `) +- Added `new()` constructor for default configuration and `new_with()` for custom configuration +- Added `matches()` and `matches_with()` methods returning results ranked by match quality and length +- Implemented thread-safe design with `Send` and `Sync` traits for concurrent usage +- Added zero-copy string storage using pointer-based indexing for memory efficiency +- Implemented smart trigram selection algorithm that samples trigrams at strategic positions (first, last, middle, then alternating) for efficient fuzzy matching +- Added interactive autocomplete example demonstrating product search functionality ([source](https://github.com/nym21/quickmatch/blob/v0.2.0/examples/autocomplete.rs)) + +[View changes](https://github.com/nym21/quickmatch/compare/e310f20...v0.2.0) diff --git a/crates/quickmatch/docs/LICENSE.md b/crates/quickmatch/docs/LICENSE.md new file mode 100644 index 000000000..c091d8fd4 --- /dev/null +++ b/crates/quickmatch/docs/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 quickmatch + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/quickmatch/docs/README.md b/crates/quickmatch/docs/README.md new file mode 100644 index 000000000..84bdf4800 --- /dev/null +++ b/crates/quickmatch/docs/README.md @@ -0,0 +1,95 @@ +# quickmatch + +Fast fuzzy string matching for Rust and JavaScript. + +Built for autocomplete, command palettes, and search-as-you-type interfaces. + +[![Crates.io](https://img.shields.io/crates/v/quickmatch.svg)](https://crates.io/crates/quickmatch) +[![npm](https://img.shields.io/npm/v/quickmatch-js.svg)](https://www.npmjs.com/package/quickmatch-js) +[![Documentation](https://docs.rs/quickmatch/badge.svg)](https://docs.rs/quickmatch) + +## Install + +```bash +# rust +cargo add quickmatch + +# js +npm install quickmatch-js +``` + +## Usage + +**Rust** + +```rust +use quickmatch::{QuickMatch, QuickMatchConfig}; + +let items = vec!["file_name", "file_size", "created_at", "updated_at"]; +let qm = QuickMatch::new(&items); + +qm.matches("file name"); // ["file_name", "file_size"] +qm.matches("filename"); // ["file_name", "file_size"] (compound match) +qm.matches("filenme"); // ["file_name", "file_size"] (trigram fuzzy) + +// Custom config +let config = QuickMatchConfig::new() + .with_limit(5) + .with_trigram_budget(10) + .with_separators(&['_', '-', ' ']); +let qm = QuickMatch::new_with(&items, config); +``` + +**JavaScript** + +```js +import { QuickMatch, QuickMatchConfig } from "quickmatch-js"; + +const items = ["file_name", "file_size", "created_at", "updated_at"]; +const qm = new QuickMatch(items); + +qm.matches("file name"); // ["file_name", "file_size"] +qm.matches("filename"); // ["file_name", "file_size"] (compound match) +qm.matches("filenme"); // ["file_name", "file_size"] (trigram fuzzy) + +// Custom config +const config = new QuickMatchConfig() + .withLimit(5) + .withTrigramBudget(10) + .withSeparators("_- "); +const qm2 = new QuickMatch(items, config); +``` + +## How it works + +Queries go through three matching stages: + +1. **Word match** — query is split by separators and looked up in a word index +2. **Compound match** — adjacent words are indexed as compounds, so `hashrate` finds `hash_rate` +3. **Trigram fallback** — unknown words are matched via character trigrams for fuzzy/typo tolerance + +Results are ranked by prefix score (exact > prefix > unordered), then by trigram score, then by length. + +## Config + +All options are documented in the `QuickMatchConfig` source. Builder methods: + +| Rust | JS | Default | +|------|-----|---------| +| `with_limit(n)` | `withLimit(n)` | 100 | +| `with_trigram_budget(n)` | `withTrigramBudget(n)` | 6 | +| `with_min_score(n)` | `withMinScore(n)` | 2 | +| `with_separators(&[..])` | `withSeparators(s)` | `_- :/` | + +## Performance + +Benchmarked against ~5,000 metric names, 83 queries, averaged over 10K iterations: + +| | Avg/query | Build time | +|---|-----------|------------| +| **Rust** | ~26 us | ~40 ms | +| **JS** | ~29 us | ~30 ms | + +## License + +MIT diff --git a/crates/quickmatch/examples/autocomplete.rs b/crates/quickmatch/examples/autocomplete.rs new file mode 100644 index 000000000..0881e77bc --- /dev/null +++ b/crates/quickmatch/examples/autocomplete.rs @@ -0,0 +1,59 @@ +use std::io::{self, Write}; + +use quickmatch::QuickMatch; + +fn main() { + let products = vec![ + "Apple iPhone 15 Pro", + "Apple MacBook Pro 16", + "Apple AirPods Pro", + "Samsung Galaxy S24", + "Samsung Galaxy Tab", + "Sony PlayStation 5", + "Sony WH-1000XM5 Headphones", + "Microsoft Surface Pro", + "Microsoft Xbox Series X", + "Dell XPS 13 Laptop", + "Dell UltraSharp Monitor", + "Logitech MX Master Mouse", + "Logitech Mechanical Keyboard", + "Canon EOS R5 Camera", + "Nikon Z9 Camera", + "GoPro Hero 12", + ] + .into_iter() + .map(|s| s.to_lowercase()) + .collect::>(); + + let products_ref = products.iter().map(|s| s.as_str()).collect::>(); + + let matcher = QuickMatch::new(&products_ref); + + println!("Type to search (press Ctrl+C to exit):"); + println!("Try: 'apple', 'pro', 'laptop', 'headphones', etc.\n"); + + loop { + print!("> "); + io::stdout().flush().unwrap(); + + let mut input = String::new(); + io::stdin().read_line(&mut input).unwrap(); + let query = input.trim(); + + if query.is_empty() { + continue; + } + + let results = matcher.matches(query); + + if results.is_empty() { + println!(" No matches found\n"); + } else { + println!(" {} result(s):", results.len()); + for result in results { + println!(" • {}", result); + } + println!(); + } + } +} diff --git a/crates/quickmatch/examples/colon_test.rs b/crates/quickmatch/examples/colon_test.rs new file mode 100644 index 000000000..bba8118a0 --- /dev/null +++ b/crates/quickmatch/examples/colon_test.rs @@ -0,0 +1,29 @@ +use quickmatch::{QuickMatch, QuickMatchConfig}; + +fn main() { + // Case 1: Items with colon, query with colon (default separators) + let items = vec!["word: word2", "other: thing", "word word2"]; + let qm = QuickMatch::new(&items); + let results = qm.matches("word: word2"); + println!("Default seps | query 'word: word2' vs items with colon: {results:?}"); + + // Case 2: Query "word:" against items + let items2 = vec!["word: word2", "word word2", "word", "word:"]; + let qm2 = QuickMatch::new(&items2); + let results2 = qm2.matches("word:"); + println!("Default seps | query 'word:' vs mixed items: {results2:?}"); + + // Case 3: Colon as separator, query "word: word2" + let config3 = QuickMatchConfig::new().with_separators(&['_', '-', ' ', ':']); + let items3 = vec!["word: word2", "other: thing", "word word2"]; + let qm3 = QuickMatch::new_with(&items3, config3); + let results3 = qm3.matches("word: word2"); + println!("Colon as sep | query 'word: word2': {results3:?}"); + + // Case 4: Colon as separator, query "word:" + let config4 = QuickMatchConfig::new().with_separators(&['_', '-', ' ', ':']); + let items4 = vec!["word: word2", "word word2", "word", "word:"]; + let qm4 = QuickMatch::new_with(&items4, config4); + let results4 = qm4.matches("word:"); + println!("Colon as sep | query 'word:' vs mixed items: {results4:?}"); +} diff --git a/crates/quickmatch/examples/compare.rs b/crates/quickmatch/examples/compare.rs new file mode 100644 index 000000000..2db054afd --- /dev/null +++ b/crates/quickmatch/examples/compare.rs @@ -0,0 +1,111 @@ +use quickmatch::{QuickMatch, QuickMatchConfig}; + +fn main() { + let content = include_str!("../metrics.txt"); + let metrics: Vec<&str> = content.lines().collect(); + let qm = QuickMatch::new(&metrics); + let config = QuickMatchConfig::new().with_limit(10); + + let queries = [ + "price", + "price close", + "price-close", + "realized price", + "supply", + "mvrv", + "sopr", + "sth", + "lth", + "utxo", + "hash rate", + "market cap", + "realized cap", + "tx count", + "fee rate", + "block size", + "hashrate", + "marketcap", + "feerate", + "blocksize", + "pric", + "suply", + "realizd", + "hashrat", + "mvr", + "real", + "profit", + "loss", + "vol", + "cap", + "sent", + "active", + "dormant", + "sth supply", + "lth realized", + "p2pkh supply", + "p2tr sent", + "nupl", + "nvt", + "cvdd", + "puell", + "rhodl", + "difficulty", + "coinbase", + "vbytes", + "segwit", + "taproot", + "short term holder", + "long term holder supply", + "realized market cap", + "unrealized profit", + "net unrealized", + "bitcoin price usd", + "total supply", + "number of transactions", + "transaction count", + "mining revenue", + "block reward", + "average fee", + "median fee rate", + "MarketCap", + "MVRV", + "HashRate", + "SOPR", + "NUPL", + "hodl", + "pnl", + "roi", + "ath", + "drawdown", + "volatility", + "dominance", + "inflation", + "velocity", + "thermocap", + "sma 200", + "sma200", + "30d", + "1year", + "return 1y", + "realised", + "sth/supply", + "lth.mvrv", + "dom", + "sub dom", + "fee dom", + "do", + "d", + "s", + "sma_200", + "200 sma", + "cap market", + "rate hash", + "supply xyz", + "xyz abc", + ]; + + for query in &queries { + let results = qm.matches_with(query, &config); + println!("{}\t{}", query, results.join(",")); + } +} diff --git a/crates/quickmatch/src/config.rs b/crates/quickmatch/src/config.rs new file mode 100644 index 000000000..60d15c1b5 --- /dev/null +++ b/crates/quickmatch/src/config.rs @@ -0,0 +1,85 @@ +const DEFAULT_SEPARATORS: &[char] = &['_', '-', ' ', ':', '/']; +const DEFAULT_TRIGRAM_BUDGET: usize = 6; +const DEFAULT_LIMIT: usize = 100; +const DEFAULT_MIN_SCORE: usize = 2; + +pub struct QuickMatchConfig { + /// Separators used to split words. + /// + /// Default: ['_', '-', ' ', ':', '/'] + separators: &'static [char], + /// Maximum number of results to return. + /// + /// Default: 100 + /// - Min: 1 + /// - Max: No hard limit (but large values may impact performance) + limit: usize, + /// Budget of trigrams to process from unknown words. + /// This budget is distributed fairly across all unknown words. + /// + /// Default: 6 (recommended: 3-9) + /// - 0: Disable trigram matching (only exact word matches) + /// - Low (3-6): Faster, less accurate fuzzy matching + /// - High (9-15): Slower, more accurate fuzzy matching + /// - Max: 20 + trigram_budget: usize, + /// Minimum trigram score required for fuzzy matches. + /// Higher values require more trigram overlap, reducing noise. + /// + /// Default: 2 + /// - Min: 1 + min_score: usize, +} + +impl Default for QuickMatchConfig { + fn default() -> Self { + Self { + separators: DEFAULT_SEPARATORS, + limit: DEFAULT_LIMIT, + trigram_budget: DEFAULT_TRIGRAM_BUDGET, + min_score: DEFAULT_MIN_SCORE, + } + } +} + +impl QuickMatchConfig { + pub fn new() -> Self { + Self::default() + } + + pub fn with_limit(mut self, limit: usize) -> Self { + self.limit = limit.max(1); + self + } + + pub fn with_trigram_budget(mut self, trigram_budget: usize) -> Self { + self.trigram_budget = trigram_budget.clamp(0, 20); + self + } + + pub fn with_separators(mut self, separators: &'static [char]) -> Self { + self.separators = separators; + self + } + + pub fn with_min_score(mut self, min_score: usize) -> Self { + self.min_score = min_score.max(1); + self + } + + pub fn limit(&self) -> usize { + self.limit + } + + pub fn trigram_budget(&self) -> usize { + self.trigram_budget + } + + pub fn separators(&self) -> &[char] { + self.separators + } + + pub fn min_score(&self) -> usize { + self.min_score + } +} diff --git a/crates/quickmatch/src/lib.rs b/crates/quickmatch/src/lib.rs new file mode 100644 index 000000000..0ab41a639 --- /dev/null +++ b/crates/quickmatch/src/lib.rs @@ -0,0 +1,381 @@ +use std::{iter, marker::PhantomData}; + +use rustc_hash::{FxHashMap, FxHashSet}; + +mod config; + +pub use config::*; + +/// Instant search over a list of strings. +/// +/// Supports exact words, prefixes ("dom" → "dominance"), joined words +/// ("hashrate" → "hash_rate"), and typo tolerance ("suply" → "supply"). +/// Results are ranked: exact matches first, then by specificity. +pub struct QuickMatch<'a> { + config: QuickMatchConfig, + max_word_count: usize, + max_word_len: usize, + max_query_len: usize, + word_index: FxHashMap>, + trigram_index: FxHashMap<[char; 3], FxHashSet<*const str>>, + _phantom: PhantomData<&'a str>, +} + +unsafe impl Send for QuickMatch<'_> {} +unsafe impl Sync for QuickMatch<'_> {} + +impl<'a> QuickMatch<'a> { + /// Expect the items to be pre-formatted (lowercase) + pub fn new(items: &[&'a str]) -> Self { + Self::new_with(items, QuickMatchConfig::default()) + } + + /// Expect the items to be pre-formatted (lowercase) + pub fn new_with(items: &[&'a str], config: QuickMatchConfig) -> Self { + let mut word_index: FxHashMap> = FxHashMap::default(); + let mut trigram_index: FxHashMap<[char; 3], FxHashSet<*const str>> = FxHashMap::default(); + let mut max_word_len = 0; + let mut max_query_len = 0; + let mut max_words = 0; + let sep = sep_table(config.separators()); + + for &item in items { + max_query_len = max_query_len.max(item.len()); + let item_words: Vec<&str> = words(item, &sep).collect(); + max_words = max_words.max(item_words.len()); + + for word in &item_words { + max_word_len = max_word_len.max(word.len()); + + for len in 1..=word.len() { + word_index + .entry(word[..len].to_string()) + .or_default() + .insert(item); + } + + let mut chars = word.chars(); + if let (Some(mut a), Some(mut b)) = (chars.next(), chars.next()) { + for c in chars { + trigram_index.entry([a, b, c]).or_default().insert(item); + a = b; + b = c; + } + } + } + + for pair in item_words.windows(2) { + let compound = format!("{}{}", pair[0], pair[1]); + // A joined-word query ("hashrate") can be longer than any + // single word. Capping at the longest index key keeps the + // DDoS guard data-bounded while still letting it match. + max_word_len = max_word_len.max(compound.len()); + let from = pair[0].len() + 1; + for len in from..=compound.len() { + word_index + .entry(compound[..len].to_string()) + .or_default() + .insert(item); + } + } + } + + Self { + max_query_len: max_query_len + 6, + max_word_len: max_word_len + 4, + max_word_count: max_words + 2, + word_index, + trigram_index, + config, + _phantom: PhantomData, + } + } + + pub fn matches(&self, query: &str) -> Vec<&'a str> { + self.matches_with(query, &self.config) + } + + pub fn matches_with(&self, query: &str, config: &QuickMatchConfig) -> Vec<&'a str> { + let limit = config.limit(); + let trigram_budget = config.trigram_budget(); + + let query: String = query + .trim() + .chars() + .filter(|c| c.is_ascii()) + .map(|c| c.to_ascii_lowercase()) + .collect(); + + if query.is_empty() || query.len() > self.max_query_len { + return vec![]; + } + + let sep = sep_table(config.separators()); + + let mut query_words: Vec<&str> = vec![]; + for w in words(&query, &sep) { + if w.len() <= self.max_word_len && !query_words.contains(&w) { + query_words.push(w); + } + } + + if query_words.is_empty() || query_words.len() > self.max_word_count { + return vec![]; + } + + let mut unknown_words: Vec<&str> = vec![]; + let mut known_sets: Vec<&FxHashSet<*const str>> = vec![]; + + for &word in &query_words { + if let Some(items) = self.word_index.get(word) { + known_sets.push(items) + } else if word.len() >= 3 && unknown_words.len() < trigram_budget { + unknown_words.push(word) + } + } + + let pool = Self::intersect_sets(&known_sets); + + // Try typo matching for unknown words + if !unknown_words.is_empty() && trigram_budget > 0 { + let min_len = query.len().saturating_sub(3); + let (scores, hit_count) = + self.score_trigrams(&unknown_words, trigram_budget, pool.as_ref(), min_len); + let min_score = hit_count.div_ceil(2).max(config.min_score()); + let results = Self::rank( + scores.into_iter().filter(|(_, s)| *s >= min_score), + &query_words, + &sep, + limit, + ); + + if !results.is_empty() { + return results; + } + } + + // Rank known candidates (intersection, or union as fallback) + let candidates = pool.unwrap_or_else(|| Self::union_sets(&known_sets)); + Self::rank( + candidates.into_iter().map(|p| (p, 0)), + &query_words, + &sep, + limit, + ) + } + + /// Intersection of all sets, or `None` when there are no sets or no + /// overlap. Clones the smallest set, then narrows it against the rest; + /// the clone's own source set is skipped since it would change nothing. + fn intersect_sets(sets: &[&FxHashSet<*const str>]) -> Option> { + let (smallest_idx, smallest) = sets + .iter() + .copied() + .enumerate() + .min_by_key(|(_, s)| s.len())?; + let mut result = smallest.clone(); + + for (i, set) in sets.iter().enumerate() { + if i == smallest_idx { + continue; + } + result.retain(|ptr| set.contains(ptr)); + if result.is_empty() { + return None; + } + } + + Some(result) + } + + /// Union of all sets. + fn union_sets(sets: &[&FxHashSet<*const str>]) -> FxHashSet<*const str> { + sets.iter().flat_map(|s| s.iter().copied()).collect() + } + + /// Bucket by matched-word count, then sort each needed bucket by fuzzy + /// score, match position, and length. + fn rank( + candidates: impl IntoIterator, + query_words: &[&str], + sep: &[bool; 256], + limit: usize, + ) -> Vec<&'a str> { + let mut buckets: Vec> = vec![vec![]; query_words.len() + 1]; + + for (item, fuzzy) in candidates { + let s = unsafe { &*item as &'a str }; + let (matched, position) = word_match(s, query_words, sep); + buckets[matched].push((s, fuzzy, position)); + } + + let mut results = Vec::with_capacity(limit); + for bucket in buckets.iter_mut().rev() { + if bucket.is_empty() { + continue; + } + bucket.sort_unstable_by(|a, b| { + b.1.cmp(&a.1) // fuzzy score, desc + .then(a.2.cmp(&b.2)) // match position, asc + .then(a.0.len().cmp(&b.0.len())) // item length, asc + .then(a.0.cmp(b.0)) // item text, asc (total order) + }); + results.extend(bucket.iter().take(limit - results.len()).map(|&(s, ..)| s)); + if results.len() >= limit { + break; + } + } + + results + } + + /// Builds per-item trigram-overlap scores for the unknown (typo) words. + /// With a `pool`, only pooled items can score (each pre-seeded to 1); + /// otherwise any item at least `min_len` chars long is eligible. Returns + /// the score map and how many probed trigrams were found in the index. + fn score_trigrams( + &self, + unknown_words: &[&str], + trigram_budget: usize, + pool: Option<&FxHashSet<*const str>>, + min_len: usize, + ) -> (FxHashMap<*const str, usize>, usize) { + let mut scores: FxHashMap<*const str, usize> = FxHashMap::default(); + scores.reserve(256); + if let Some(pool) = pool { + for &item in pool { + scores.insert(item, 1); + } + } + let has_pool = pool.is_some(); + + let mut budget = trigram_budget; + let mut hit_count = 0; + let mut visited: FxHashSet<[char; 3]> = FxHashSet::default(); + + 'outer: for round in 0..trigram_budget { + for word in unknown_words { + if budget == 0 { + break 'outer; + } + + let bytes = word.as_bytes(); + let Some(pos) = trigram_position(bytes.len(), round) else { + continue; + }; + let trigram = [ + bytes[pos] as char, + bytes[pos + 1] as char, + bytes[pos + 2] as char, + ]; + + if !visited.insert(trigram) { + continue; + } + budget -= 1; + + let Some(items) = self.trigram_index.get(&trigram) else { + continue; + }; + hit_count += 1; + + if has_pool { + for &item in items { + if let Some(score) = scores.get_mut(&item) { + *score += 1; + } + } + } else { + for &item in items { + if unsafe { &*item }.len() >= min_len { + *scores.entry(item).or_default() += 1; + } + } + } + } + } + + (scores, hit_count) + } +} + +/// Builds a byte lookup table from the configured separator chars. Separators +/// are ASCII, so a byte-indexed table is exact even for multi-byte UTF-8: +/// continuation and lead bytes are all >= 128 and never flagged. +fn sep_table(separators: &[char]) -> [bool; 256] { + let mut table = [false; 256]; + for &c in separators { + if (c as usize) < 256 { + table[c as usize] = true; + } + } + table +} + +/// Splits `text` into non-empty words on any separator byte flagged in `sep`. +fn words<'s>(text: &'s str, sep: &'s [bool; 256]) -> impl Iterator { + let bytes = text.as_bytes(); + let mut i = 0; + iter::from_fn(move || { + while i < bytes.len() && sep[bytes[i] as usize] { + i += 1; + } + let start = i; + while i < bytes.len() && !sep[bytes[i] as usize] { + i += 1; + } + (i > start).then(|| &text[start..i]) + }) +} + +/// Aligns the query words against the item's words, in order: +/// - `matched`: query words matched as an in-order subsequence of item words +/// - `position`: index of the item word where that run starts (or the item's +/// word count when nothing matched) +fn word_match(item: &str, query_words: &[&str], sep: &[bool; 256]) -> (usize, usize) { + let mut matched = 0; + let mut position = 0; + for iw in words(item, sep) { + if query_words + .get(matched) + .is_some_and(|qw| iw.starts_with(*qw)) + { + matched += 1; + } else if matched == 0 { + position += 1; + } + } + (matched, position) +} + +/// Picks which trigram of a length-`len` word to probe on `round`, spreading +/// probes outward from the two ends toward the middle. Returns `None` when the +/// round offers no fresh position. +fn trigram_position(len: usize, round: usize) -> Option { + let max = len - 3; + if round == 0 { + return Some(0); + } + if round == 1 && max > 0 { + return Some(max); + } + if round == 2 && max > 1 { + return Some(max / 2); + } + if max <= 2 { + return None; + } + + let mid = max / 2; + let offset = (round - 2) >> 1; + let pos = if round & 1 == 1 { + mid.saturating_sub(offset) + } else { + mid + offset + }; + if pos == 0 || pos >= max || pos == mid { + None + } else { + Some(pos) + } +} diff --git a/crates/rawdb/Cargo.toml b/crates/rawdb/Cargo.toml new file mode 100644 index 000000000..648f226d0 --- /dev/null +++ b/crates/rawdb/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rawdb" +description = "Single-file, low-level and space efficient storage engine with filesystem-like API" +keywords = ["vec", "disk", "database"] +categories = ["database"] +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +readme.workspace = true + +[dependencies] +libc = { workspace = true } +log = { workspace = true } +memmap2 = "0.9.11" +parking_lot = { workspace = true } +rayon = { workspace = true } +smallvec = "1.15" +thiserror = "2.0.18" + +[dev-dependencies] +tempfile = { workspace = true } + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/rawdb/README.md b/crates/rawdb/README.md new file mode 100644 index 000000000..1f2751d2b --- /dev/null +++ b/crates/rawdb/README.md @@ -0,0 +1,96 @@ +# rawdb + +Non-transactional embedded storage engine with a filesystem-like API. + +It features: + +- Multiple named regions in one file +- Automatic space reclamation via hole punching +- Regions grow and move automatically as needed +- Zero-copy mmap access +- Thread-safe with concurrent reads and writes +- Page-aligned allocations (4KB) +- Crash-consistent with explicit flush +- Foundation for higher-level abstractions (e.g., [`vecdb`](../vecdb/README.md)) + +It is not: + +- A transactional database (no ACID, transactions, or rollback) +- A query engine (no SQL, indexes, or schemas) + +## Install + +```bash +cargo add rawdb +``` + +## Usage + +```rust +use rawdb::{Database, Result}; + +fn main() -> Result<()> { + // open database + let temp_dir = tempfile::TempDir::new()?; + let db = Database::open(temp_dir.path())?; + + // create regions + let region1 = db.create_region_if_needed("region1")?; + let region2 = db.create_region_if_needed("region2")?; + + // write data (buffered in mmap, not yet durable) + region1.write(&[0, 1, 2, 3, 4])?; + region2.write_at(&[5, 6, 7, 8, 9], 0)?; + + // flush to disk for durability + db.flush()?; + + // read via mmap (data is immediately visible) + { + let reader = region1.create_reader(); + let _data = reader.read_all(); + } // reader dropped here, releasing its reference + + // remove region (space becomes reusable hole after flush) + region1.remove()?; + + // punch holes in the file + // db.compact()?; // doesn't work with doc-tests + + Ok(()) +} +``` + +## Sparse files + +rawdb relies on **sparse file** support. Files grow via `set_len()` which creates logical size without allocating physical blocks, and `compact()` punches holes to reclaim unused blocks. This means: + +- The filesystem must support sparse files (ext4, XFS, APFS, ZFS, Btrfs — most modern filesystems do) +- `du` and `ls -s` show actual disk usage; `ls -l` shows the larger logical size +- Copying with `cp` may "densify" the file unless you use `cp --sparse=always` (Linux) or a sparse-aware tool +- Backups should use sparse-aware tools to avoid inflating archives + +## Durability + +Operations become durable after calling `flush()`. Before flush, writes are visible in memory but not guaranteed to survive crashes. + +**Design:** +- **4KB metadata entries**: Atomic page-sized writes per region with embedded IDs +- **Single metadata file**: Rebuilt into HashMap on startup for O(1) lookups +- **No WAL**: Simple design with lazy flushing for consistency +- **Lazy writes**: Both data and metadata are written to mmaps immediately but not synced until flush + +**Write model:** +1. **Data writes** modify the data mmap immediately (visible but not durable) +2. **Metadata changes** modify the metadata mmap immediately (visible but not durable) +3. **Holes from moves/removes** are marked as pending (not reusable until flush) +4. **`flush()`** syncs both mmaps (data → metadata → file size), then promotes pending holes +5. Ensures metadata never points to unflushed data and old locations aren't reused prematurely + +**Region operations:** +- Expand in-place when possible (last region or adjacent hole) +- Copy-on-write to new location when expansion needed +- All changes visible immediately in mmaps, durable after `flush()` + +**Recovery:** +On open, reads all metadata entries and rebuilds in-memory structures. Deleted regions are identified by zeroed metadata. diff --git a/crates/rawdb/src/disk_usage.rs b/crates/rawdb/src/disk_usage.rs new file mode 100644 index 000000000..120f350f7 --- /dev/null +++ b/crates/rawdb/src/disk_usage.rs @@ -0,0 +1,50 @@ +use std::fs::File; + +use crate::Result; + +/// Actual disk usage (accounts for sparse files / holes). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DiskUsage(u64); + +impl DiskUsage { + #[cfg(unix)] + pub fn from_file(file: &File) -> Result { + use std::os::unix::io::AsRawFd; + + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + let result = unsafe { libc::fstat(file.as_raw_fd(), &mut stat) }; + if result == -1 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(Self(stat.st_blocks as u64 * 512)) + } + + #[cfg(not(unix))] + pub fn from_file(file: &File) -> Result { + Ok(Self(file.metadata()?.len())) + } + + #[inline] + pub fn bytes(&self) -> u64 { + self.0 + } +} + +impl std::fmt::Display for DiskUsage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + const KIB: u64 = 1024; + const MIB: u64 = KIB * 1024; + const GIB: u64 = MIB * 1024; + + let bytes = self.0; + if bytes >= GIB { + write!(f, "{:.1} GiB", bytes as f64 / GIB as f64) + } else if bytes >= MIB { + write!(f, "{:.1} MiB", bytes as f64 / MIB as f64) + } else if bytes >= KIB { + write!(f, "{:.1} KiB", bytes as f64 / KIB as f64) + } else { + write!(f, "{} B", bytes) + } + } +} diff --git a/crates/rawdb/src/error.rs b/crates/rawdb/src/error.rs new file mode 100644 index 000000000..eb8c47a10 --- /dev/null +++ b/crates/rawdb/src/error.rs @@ -0,0 +1,88 @@ +use std::{fs, io, result}; + +use thiserror::Error; + +pub type Result = result::Result; + +/// Error types for rawdb operations. +#[derive(Debug, Error)] +pub enum Error { + #[error(transparent)] + IO(#[from] io::Error), + + #[error("Database is locked by another process")] + TryLock(#[from] fs::TryLockError), + + // Region errors + #[error("Region not found")] + RegionNotFound, + + #[error("Region metadata has unwritten data")] + RegionMetadataUnwritten, + + #[error("Region already exists")] + RegionAlreadyExists, + + #[error("Cannot remove region '{id}': still held by {} reference(s)", ref_count - 1)] + RegionStillReferenced { id: String, ref_count: usize }, + + // Write errors + #[error("Write position {position} is beyond region length {region_len}")] + WriteOutOfBounds { position: usize, region_len: usize }, + + // Truncate errors + #[error("Cannot truncate to {from} bytes (current length: {current_len})")] + TruncateInvalid { from: usize, current_len: usize }, + + // Metadata errors + #[error("Invalid region ID")] + InvalidRegionId, + + #[error("Invalid metadata size: expected {expected} bytes, got {actual}")] + InvalidMetadataSize { expected: usize, actual: usize }, + + #[error("Empty region metadata")] + EmptyMetadata, + + // Layout errors + #[error("Region index mismatch in layout")] + RegionIndexMismatch, + + #[error("Hole too small: have {hole_size} bytes, need {requested}")] + HoleTooSmall { hole_size: usize, requested: usize }, + + // Internal invariant errors + #[error("Internal invariant violated: {0}")] + InvariantViolation(String), + + #[error("Corrupted metadata: {0}")] + CorruptedMetadata(String), + + #[error("Region size would overflow: current={current}, requested={requested}")] + RegionSizeOverflow { current: usize, requested: usize }, + + #[error("Overlapping copy ranges not supported (src={src}..{src_end}, dst={dst}..{dst_end})")] + OverlappingCopyRanges { + src: usize, + src_end: usize, + dst: usize, + dst_end: usize, + }, + + // Hole punching errors + #[error("Failed to punch hole at offset {start} (length {len}): {source}")] + HolePunchFailed { + start: usize, + len: usize, + source: io::Error, + }, + + #[error("Hole punching is not supported on this platform")] + HolePunchUnsupported, +} + +impl Error { + pub fn other(e: impl ToString) -> Self { + Self::InvariantViolation(e.to_string()) + } +} diff --git a/crates/rawdb/src/hints.rs b/crates/rawdb/src/hints.rs new file mode 100644 index 000000000..a76217913 --- /dev/null +++ b/crates/rawdb/src/hints.rs @@ -0,0 +1,19 @@ +#[inline(always)] +pub fn likely(b: bool) -> bool { + if !b { + cold(); + } + b +} + +#[inline(always)] +pub fn unlikely(b: bool) -> bool { + if b { + cold(); + } + b +} + +#[inline(always)] +#[cold] +fn cold() {} diff --git a/crates/rawdb/src/hole_punch.rs b/crates/rawdb/src/hole_punch.rs new file mode 100644 index 000000000..433786482 --- /dev/null +++ b/crates/rawdb/src/hole_punch.rs @@ -0,0 +1,108 @@ +use std::fs::File; + +#[cfg(unix)] +use std::os::unix::io::AsRawFd; + +use crate::{Error, Result}; + +/// Deallocates file blocks without changing file size (platform-specific). +pub struct HolePunch; + +impl HolePunch { + #[cfg(target_os = "macos")] + pub fn punch(file: &File, start: usize, length: usize) -> Result<()> { + let fpunchhole = FPunchhole { + fp_flags: 0, + reserved: 0, + fp_offset: start as libc::off_t, + fp_length: length as libc::off_t, + }; + + let result = unsafe { + libc::fcntl( + file.as_raw_fd(), + libc::F_PUNCHHOLE, + &fpunchhole as *const FPunchhole, + ) + }; + + if result == -1 { + let err = std::io::Error::last_os_error(); + return Err(Error::HolePunchFailed { + start, + len: length, + source: err, + }); + } + + Ok(()) + } + + #[cfg(target_os = "linux")] + pub fn punch(file: &File, start: usize, length: usize) -> Result<()> { + let result = unsafe { + libc::fallocate( + file.as_raw_fd(), + libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE, + start as libc::off_t, + length as libc::off_t, + ) + }; + + if result == -1 { + let err = std::io::Error::last_os_error(); + return Err(Error::HolePunchFailed { + start, + len: length, + source: err, + }); + } + + Ok(()) + } + + #[cfg(target_os = "freebsd")] + pub fn punch(file: &File, start: usize, length: usize) -> Result<()> { + let fd = file.as_raw_fd(); + + let mut spacectl = libc::spacectl_range { + r_offset: start as libc::off_t, + r_len: length as libc::off_t, + }; + + let result = unsafe { + libc::fspacectl( + fd, + libc::SPACECTL_DEALLOC, + &spacectl as *const libc::spacectl_range, + 0, + &mut spacectl as *mut libc::spacectl_range, + ) + }; + + if result == -1 { + let err = std::io::Error::last_os_error(); + return Err(Error::HolePunchFailed { + start, + len: length, + source: err, + }); + } + + Ok(()) + } + + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "freebsd")))] + pub fn punch(_file: &File, _start: usize, _length: usize) -> Result<()> { + Err(Error::HolePunchUnsupported) + } +} + +#[cfg(target_os = "macos")] +#[repr(C)] +struct FPunchhole { + fp_flags: u32, + reserved: u32, + fp_offset: libc::off_t, + fp_length: libc::off_t, +} diff --git a/crates/rawdb/src/layout.rs b/crates/rawdb/src/layout.rs new file mode 100644 index 000000000..242b9aa71 --- /dev/null +++ b/crates/rawdb/src/layout.rs @@ -0,0 +1,226 @@ +use std::{collections::BTreeMap, mem}; + +use log::debug; +use smallvec::SmallVec; + +use crate::{Error, Region, Regions, Result}; + +/// Tracks regions, holes, and reservations in the database file. +#[derive(Debug, Default)] +pub struct Layout { + start_to_region: BTreeMap, + start_to_hole: BTreeMap, + /// hole_size → starts: enables O(log n) best-fit search. + hole_to_starts: BTreeMap>, + start_to_reserved: BTreeMap, + /// Holes from region moves, reusable after flush. + pending_holes: BTreeMap, +} + +impl From<&Regions> for Layout { + fn from(regions: &Regions) -> Self { + let start_to_region: BTreeMap = regions + .index_to_region() + .iter() + .flatten() + .map(|region| (region.meta().start(), region.clone())) + .collect(); + + let mut layout = Self { + start_to_hole: BTreeMap::default(), + hole_to_starts: BTreeMap::default(), + start_to_reserved: BTreeMap::default(), + pending_holes: BTreeMap::default(), + start_to_region: BTreeMap::default(), + }; + + let mut prev_end = 0; + for (&start, region) in &start_to_region { + if prev_end != start { + let size = start - prev_end; + layout.insert_hole(prev_end, size); + } + prev_end = start + region.meta().reserved(); + } + + layout.start_to_region = start_to_region; + layout + } +} + +impl Layout { + fn insert_hole(&mut self, start: usize, size: usize) { + self.start_to_hole.insert(start, size); + self.hole_to_starts.entry(size).or_default().push(start); + } + + fn remove_hole(&mut self, start: usize) -> Option { + let size = self.start_to_hole.remove(&start)?; + + if let Some(starts) = self.hole_to_starts.get_mut(&size) { + starts.retain(|s| *s != start); + if starts.is_empty() { + self.hole_to_starts.remove(&size); + } + } + + Some(size) + } + + pub fn start_to_region(&self) -> &BTreeMap { + &self.start_to_region + } + + pub fn start_to_hole(&self) -> &BTreeMap { + &self.start_to_hole + } + + pub fn len(&self) -> usize { + let mut len = 0; + if let Some((start, reserved)) = self.get_last_reserved() { + len = len.max(start + reserved); + } + if let Some((start, gap)) = self.get_last_hole() { + len = len.max(start + gap); + } + if let Some((&start, &size)) = self.pending_holes.last_key_value() { + len = len.max(start + size); + } + if let Some((start, region)) = self.get_last_region() { + len = len.max(start + region.meta().reserved()); + } + len + } + + pub fn get_last_region(&self) -> Option<(usize, &Region)> { + self.start_to_region + .last_key_value() + .map(|(start, region)| (*start, region)) + } + + fn get_last_hole(&self) -> Option<(usize, usize)> { + self.start_to_hole + .last_key_value() + .map(|(start, gap)| (*start, *gap)) + } + + fn get_last_reserved(&self) -> Option<(usize, usize)> { + self.start_to_reserved + .last_key_value() + .map(|(start, reserved)| (*start, *reserved)) + } + + pub fn is_last_anything(&self, region: &Region) -> bool { + let Some((last_start, last_region)) = self.get_last_region() else { + return false; + }; + + last_region.index() == region.index() + && self + .get_last_hole() + .is_none_or(|(hole_start, _)| last_start > hole_start) + && self + .get_last_reserved() + .is_none_or(|(reserved_start, _)| last_start > reserved_start) + && self + .pending_holes + .last_key_value() + .is_none_or(|(&pending_start, _)| last_start > pending_start) + } + + pub fn insert_region(&mut self, start: usize, region: &Region) { + assert!(self.start_to_region.insert(start, region.clone()).is_none()) + } + + pub fn move_region(&mut self, new_start: usize, region: &Region) -> Result<()> { + self.remove_region(region)?; + self.insert_region(new_start, region); + Ok(()) + } + + pub fn remove_region(&mut self, region: &Region) -> Result<()> { + let region_meta = region.meta(); + let start = region_meta.start(); + let reserved = region_meta.reserved(); + + let removed = self.start_to_region.remove(&start); + + if removed + .as_ref() + .is_none_or(|region_| region.index() != region_.index()) + { + return Err(Error::RegionIndexMismatch); + } + + self.pending_holes.insert(start, reserved); + + Ok(()) + } + + pub fn get_hole(&self, start: usize) -> Option { + self.start_to_hole.get(&start).copied() + } + + pub fn find_smallest_adequate_hole(&self, min_size: usize) -> Option { + self.hole_to_starts + .range(min_size..) + .next() + .and_then(|(_, starts)| starts.first().copied()) + } + + pub fn remove_or_compress_hole(&mut self, start: usize, compress_by: usize) -> Result<()> { + let Some(size) = self.remove_hole(start) else { + return Ok(()); + }; + + if size == compress_by { + Ok(()) + } else if size > compress_by { + let new_start = start + compress_by; + let new_size = size - compress_by; + self.insert_hole(new_start, new_size); + Ok(()) + } else { + Err(Error::HoleTooSmall { + hole_size: size, + requested: compress_by, + }) + } + } + + pub fn reserve(&mut self, start: usize, reserved: usize) { + if self.start_to_reserved.insert(start, reserved).is_some() { + unreachable!(); + } + } + + pub fn take_reserved(&mut self, start: usize) -> Option { + self.start_to_reserved.remove(&start) + } + + pub fn promote_pending_holes(&mut self, name: &str) { + let count = self.pending_holes.len(); + if count > 0 { + debug!("{}: promoted {} pending holes", name, count); + } + for (start, mut size) in mem::take(&mut self.pending_holes) { + let mut final_start = start; + + // Coalesce with adjacent real hole BEFORE + if let Some((&hole_start, &hole_size)) = self.start_to_hole.range(..start).next_back() + && hole_start + hole_size == start + { + self.remove_hole(hole_start); + final_start = hole_start; + size += hole_size; + } + + // Coalesce with adjacent real hole AFTER + if let Some(hole_after_size) = self.remove_hole(final_start + size) { + size += hole_after_size; + } + + self.insert_hole(final_start, size); + } + } +} diff --git a/crates/rawdb/src/lib.rs b/crates/rawdb/src/lib.rs new file mode 100644 index 000000000..9e5cb4bea --- /dev/null +++ b/crates/rawdb/src/lib.rs @@ -0,0 +1,666 @@ +#![doc = include_str!("../README.md")] + +use std::{ + collections::HashSet, + fmt, + fs::{self, File, OpenOptions}, + path::{Path, PathBuf}, + sync::{ + Arc, Weak, + atomic::{AtomicUsize, Ordering}, + }, + thread::{self, JoinHandle}, + time::{Duration, Instant}, +}; + +use log::{debug, trace}; +use memmap2::MmapMut; +use parking_lot::{Condvar, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +mod disk_usage; +pub mod error; +mod hints; +mod hole_punch; +mod layout; +mod mmap; +mod reader; +mod region; +mod region_metadata; +mod region_state; +mod regions; + +pub use disk_usage::*; +pub use error::*; +pub use hints::*; +use hole_punch::*; +use layout::*; +use mmap::*; +use rayon::prelude::*; +pub use reader::*; +pub use region::*; +pub use region_metadata::*; +use region_state::*; +use regions::*; + +pub const PAGE_SIZE: usize = 4096; +pub const PAGE_SIZE_MINUS_1: usize = PAGE_SIZE - 1; +/// One gibibyte (1024^3 bytes). +#[allow(non_upper_case_globals)] +pub const GiB: usize = 1024 * 1024 * 1024; + +/// Memory-mapped database with region-based storage and hole punching. +#[derive(Clone)] +#[must_use = "Database should be stored to keep the database open"] +pub struct Database(Arc); + +/// Lock ordering: layout → regions → mmap → file → meta → dirty_bounds. +struct DatabaseInner { + path: PathBuf, + name: String, + layout: RwLock, + regions: RwLock, + mmap: RwLock, + file: RwLock, + cached_file_len: AtomicUsize, + bg_tasks: Mutex>>>, + bg_sync: (Mutex, Condvar), +} + +impl Database { + /// Opens or creates a database at `path`. + pub fn open(path: &Path) -> Result { + Self::open_with_min_len(path, 0) + } + + pub fn open_with_min_len(path: &Path, min_len: usize) -> Result { + let name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + fs::create_dir_all(path)?; + + let file = OpenOptions::new() + .read(true) + .create(true) + .write(true) + .truncate(false) + .open(Self::data_path_from(path))?; + + file.try_lock()?; + + let mut file_len = file.metadata()?.len() as usize; + if file_len < min_len { + file.set_len(min_len as u64)?; + file.sync_all()?; + file_len = min_len; + } + + let regions = Regions::open(path)?; + let mmap = create_mmap(&file)?; + + let db = Self(Arc::new(DatabaseInner { + path: path.to_owned(), + name, + layout: RwLock::new(Layout::default()), + regions: RwLock::new(regions), + mmap: RwLock::new(mmap), + file: RwLock::new(file), + cached_file_len: AtomicUsize::new(file_len), + bg_tasks: Mutex::new(Vec::new()), + bg_sync: (Mutex::new(false), Condvar::new()), + })); + + db.regions_mut().fill(&db)?; + *db.layout_mut() = Layout::from(&*db.regions()); + + debug!("{}: opened with {} regions", db, db.regions().len()); + + Ok(db) + } + + /// Cached file length (no syscall). + #[inline] + pub fn file_len(&self) -> usize { + self.0.cached_file_len.load(Ordering::Relaxed) + } + + /// Grows the file if needed (doubles size, 1 MiB floor, sparse-file friendly). + pub fn set_min_len(&self, len: usize) -> Result<()> { + let len = Self::ceil_number_to_page_size_multiple(len); + + if self.file_len() >= len { + return Ok(()); + } + + trace!("{}: set_min_len acquiring mmap_mut", self); + let mut mmap = self.mmap_mut(); + trace!("{}: set_min_len acquiring file_mut", self); + let file = self.file_mut(); + + // Re-check after acquiring lock (another thread may have grown the file). + let current_len = self.file_len(); + if current_len >= len { + return Ok(()); + } + + let target_len = + Self::ceil_number_to_page_size_multiple(len.max(current_len * 2).max(1024 * 1024)); + debug!( + "{}: set_min_len to {} (requested {})", + self, target_len, len + ); + file.set_len(target_len as u64)?; + self.0.cached_file_len.store(target_len, Ordering::Relaxed); + *mmap = create_mmap(&file)?; + Ok(()) + } + + pub fn set_min_regions(&self, regions: usize) -> Result<()> { + self.regions_mut() + .set_min_len(regions * SIZE_OF_REGION_METADATA)?; + self.set_min_len(regions * PAGE_SIZE) + } + + pub fn get_region(&self, id: &str) -> Option { + self.regions().get_from_id(id).cloned() + } + + pub fn create_region_if_needed(&self, id: &str) -> Result { + if let Some(region) = self.get_region(id) { + return Ok(region); + } + + let layout = self.layout(); + if layout.find_smallest_adequate_hole(PAGE_SIZE).is_none() { + let end = layout.len(); + drop(layout); + self.set_min_len(end + PAGE_SIZE)?; + } else { + drop(layout); + } + + debug!("{}: create_region_if_needed '{}'", self, id); + trace!( + "{}: create_region_if_needed '{}' acquiring layout_mut", + self, id + ); + let mut layout = self.layout_mut(); + trace!( + "{}: create_region_if_needed '{}' acquiring regions_mut", + self, id + ); + let mut regions = self.regions_mut(); + + if let Some(region) = regions.get_from_id(id).cloned() { + return Ok(region); + } + + let start = if let Some(start) = layout.find_smallest_adequate_hole(PAGE_SIZE) { + layout.remove_or_compress_hole(start, PAGE_SIZE)?; + start + } else { + layout.len() + }; + + let region = regions.create(self, id.to_owned(), start)?; + layout.insert_region(start, ®ion); + Ok(region) + } + + #[inline] + pub(crate) fn write(&self, start: usize, data: &[u8]) { + write_to_mmap(&self.mmap(), start, data); + } + + pub(crate) fn copy(&self, src: usize, dst: usize, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + let src_end = src + len; + let dst_end = dst + len; + if !(src_end <= dst || dst_end <= src) { + return Err(Error::OverlappingCopyRanges { + src, + src_end, + dst, + dst_end, + }); + } + + let mmap = self.mmap(); + write_to_mmap(&mmap, dst, &mmap[src..src_end]); + Ok(()) + } + + pub fn remove_region_if_exists(&self, id: &str) -> Result<()> { + match self.remove_region(id) { + Ok(()) | Err(Error::RegionNotFound) => Ok(()), + Err(e) => Err(e), + } + } + + pub fn remove_region(&self, id: &str) -> Result<()> { + let Some(region) = self.get_region(id) else { + return Err(Error::RegionNotFound); + }; + region.remove() + } + + /// Removes all regions except those in `ids`. + pub fn retain_regions(&self, mut ids: HashSet) -> Result<()> { + debug!( + "{}: retain_regions called with {} ids to keep", + self, + ids.len() + ); + + let regions = self.regions(); + let regions_to_remove: Vec<_> = regions + .id_to_index() + .keys() + .filter(|id| !ids.remove(&**id)) + .filter_map(|id| regions.get_from_id(id).cloned()) + .collect(); + drop(regions); + + if !ids.is_empty() { + debug!( + "{}: retain_regions: {} ids in retain set not found in db: {:?}", + self, + ids.len(), + ids + ); + } + + if !regions_to_remove.is_empty() { + debug!( + "{}: retain_regions removing {} regions: {:?}", + self, + regions_to_remove.len(), + regions_to_remove + .iter() + .map(|r| r.meta().id().to_string()) + .collect::>() + ); + } + + for region in regions_to_remove { + let ref_count = std::sync::Arc::strong_count(region.arc()); + debug!( + "{}: removing '{}' (arc count: {})", + self, + region.meta().id(), + ref_count + ); + region.remove()?; + } + Ok(()) + } + + /// Opens the data file read-only (for external consumers like mmap readers). + #[inline] + pub fn open_read_only_file(&self) -> Result { + File::open(self.data_path()).map_err(Error::from) + } + + pub fn disk_usage(&self) -> Result { + DiskUsage::from_file(&self.file()) + } + + /// Flushes all dirty data and metadata to disk. Returns number of flushed regions. + pub fn flush(&self) -> Result { + let dirty_regions: Vec<(Region, Option<(usize, usize)>)> = self + .regions() + .index_to_region() + .iter() + .flatten() + .filter_map(|r| { + let bounds = r.take_dirty_bounds(); + if bounds.is_some() || r.meta().needs_flush() { + Some((r.clone(), bounds)) + } else { + None + } + }) + .collect(); + + if dirty_regions.is_empty() { + debug!("{}: flush (no dirty)", self); + self.layout_mut().promote_pending_holes(self.name()); + return Ok(0); + } + + let (flush_start, flush_end) = dirty_regions + .iter() + .filter_map(|(r, bounds)| { + let (min, max) = (*bounds)?; + let region_start = r.meta().start(); + Some((region_start + min, region_start + max)) + }) + .fold((usize::MAX, 0usize), |(min_s, max_e), (s, e)| { + (min_s.min(s), max_e.max(e)) + }); + + if flush_start < flush_end { + let mmap = self.mmap(); + if let Err(e) = mmap.flush_async_range(flush_start, flush_end - flush_start) { + drop(mmap); + for (region, bounds) in dirty_regions { + if let Some((min, max)) = bounds { + region.restore_dirty_bounds(min, max); + } + } + return Err(e.into()); + } + } + + // Data must be durable before metadata (crash safety). + self.regions().flush()?; + self.file().sync_data()?; + self.regions().sync_data()?; + for (region, _) in &dirty_regions { + region.meta().mark_clean(); + } + + debug!("{}: flushed {} regions", self, dirty_regions.len()); + self.layout_mut().promote_pending_holes(self.name()); + Ok(dirty_regions.len()) + } + + /// Gives the OS time to write dirty mmap pages before fsyncing. + /// Intended for background tasks where the delay is invisible. + /// Cancellable: `sync_bg_tasks` cuts the wait short. + pub fn compact_deferred(&self, delay: Duration) -> Result<()> { + self.bg_sleep(delay); + self.compact() + } + + /// Cancellable wait for use inside `run_bg` closures. Returns + /// immediately when `sync_bg_tasks` is called. + pub fn bg_sleep(&self, dur: Duration) { + let (m, cv) = &self.0.bg_sync; + let mut g = m.lock(); + if !*g { + cv.wait_for(&mut g, dur); + } + } + + /// Like `compact_deferred` with a 5-second default delay. + pub fn compact_deferred_default(&self) -> Result<()> { + self.compact_deferred(Duration::from_secs(5)) + } + + /// Flushes, then punches holes to reclaim disk space. + #[inline] + pub fn compact(&self) -> Result<()> { + let i = Instant::now(); + self.flush()?; + let flush_time = i.elapsed(); + let i = Instant::now(); + let r = self.punch_holes(); + let punch_time = i.elapsed(); + debug!( + "{}: compact in {:?} (flush: {:?}, punch_holes: {:?})", + self, + flush_time + punch_time, + flush_time, + punch_time + ); + r + } + + /// Runs `f` on a background thread without incrementing the Arc refcount, + /// so `strong_count` reflects only real owners. + /// Call `sync_bg_tasks()` before the next write to this database. + pub fn run_bg(&self, f: impl FnOnce(&Self) -> Result<()> + Send + 'static) { + use std::mem::ManuallyDrop; + // Safety: sync_bg_tasks (called explicitly or from Drop at strong_count == 1) + // joins this thread before the Arc is deallocated. + // ManuallyDrop prevents the refcount decrement we never incremented. + let db = ManuallyDrop::new(unsafe { Self(Arc::from_raw(Arc::as_ptr(&self.0))) }); + self.0.bg_tasks.lock().push(thread::spawn(move || f(&db))); + } + + /// Wakes any `bg_sleep` waiters and joins all pending background tasks. + pub fn sync_bg_tasks(&self) -> Result<()> { + { + let (m, cv) = &self.0.bg_sync; + *m.lock() = true; + cv.notify_all(); + } + let handles: Vec<_> = self.0.bg_tasks.lock().drain(..).collect(); + for handle in handles { + handle.join().unwrap()?; + } + *self.0.bg_sync.0.lock() = false; + Ok(()) + } + + fn punch_holes(&self) -> Result<()> { + let layout = self.layout(); + + let regions_to_check: Vec = { + let regions = self.regions(); + regions + .index_to_region() + .iter() + .flatten() + .cloned() + .collect() + }; + + // Collect layout holes (protected by layout READ - can punch in parallel) + let layout_holes: Vec<(usize, usize)> = layout + .start_to_hole() + .iter() + .map(|(&start, &hole)| (start, hole)) + .collect(); + + let file = self.file(); + let mut punched = 0usize; + + // Punch region reserved space. We MUST hold meta WRITE before checking, + // because write_with does db.write() BEFORE updating meta. If we only + // acquire WRITE after checking, a concurrent write could be in progress. + for region in ®ions_to_check { + let meta = region.meta_mut(); + let rstart = meta.start(); + let len = meta.len(); + let reserved = meta.reserved(); + let ceil_len = Self::ceil_number_to_page_size_multiple(len); + + if ceil_len < reserved { + let start = rstart + ceil_len; + let hole = reserved - ceil_len; + if Self::approx_has_punchable_data(&file, start, hole) { + HolePunch::punch(&file, start, hole)?; + punched += 1; + } + } + } + + // Punch layout holes in parallel (safe - layout READ prevents allocation) + // No per-region lock needed since these are unallocated holes. + let layout_punched: usize = layout_holes + .par_iter() + .filter_map(|&(start, hole)| { + if Self::approx_has_punchable_data(&file, start, hole) { + HolePunch::punch(&file, start, hole).ok()?; + Some(1) + } else { + None + } + }) + .sum(); + punched += layout_punched; + + drop(file); + drop(layout); + + // No mmap recreation needed: KEEP_SIZE preserves file length, kernel zeroes punched pages. + if punched > 0 { + debug!("{}: punch_holes syncing after {} punches", self, punched); + let file = self.file(); + file.sync_data()?; + } + + Ok(()) + } + + /// Samples a few bytes via pread to check if a hole has non-zero data. + fn approx_has_punchable_data(file: &File, start: usize, len: usize) -> bool { + use std::os::unix::io::AsRawFd; + + assert!(start.is_multiple_of(PAGE_SIZE)); + assert!(len.is_multiple_of(PAGE_SIZE)); + + let fd = file.as_raw_fd(); + let mut buf = [0u8; 1]; + + let mut check_page = |page_start: usize| -> bool { + let n = unsafe { + libc::pread( + fd, + buf.as_mut_ptr() as *mut libc::c_void, + 1, + page_start as libc::off_t, + ) + }; + if n == 1 && buf[0] != 0 { + return true; + } + + let page_end = page_start + PAGE_SIZE - 1; + let n = unsafe { + libc::pread( + fd, + buf.as_mut_ptr() as *mut libc::c_void, + 1, + page_end as libc::off_t, + ) + }; + n == 1 && buf[0] != 0 + }; + + if check_page(start) { + return true; + } + + let last_page_start = start + len - PAGE_SIZE; + if last_page_start != start && check_page(last_page_start) { + return true; + } + + if len > GiB { + let num_gb_checks = len / GiB; + for i in 1..num_gb_checks { + let gb_boundary = start + i * GiB; + if check_page(gb_boundary) { + return true; + } + } + } + + false + } + + #[inline(always)] + pub fn file(&self) -> RwLockReadGuard<'_, File> { + self.0.file.read() + } + + #[inline(always)] + pub fn file_mut(&self) -> RwLockWriteGuard<'_, File> { + self.0.file.write() + } + + #[inline(always)] + pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> { + self.0.mmap.read() + } + + #[inline(always)] + pub fn mmap_mut(&self) -> RwLockWriteGuard<'_, MmapMut> { + self.0.mmap.write() + } + + #[inline(always)] + pub fn regions(&self) -> RwLockReadGuard<'_, Regions> { + self.0.regions.read() + } + + #[inline(always)] + pub(crate) fn regions_mut(&self) -> RwLockWriteGuard<'_, Regions> { + self.0.regions.write() + } + + #[inline(always)] + pub fn layout(&self) -> RwLockReadGuard<'_, Layout> { + self.0.layout.read() + } + + #[inline(always)] + pub(crate) fn layout_mut(&self) -> RwLockWriteGuard<'_, Layout> { + self.0.layout.write() + } + + #[inline] + fn ceil_number_to_page_size_multiple(num: usize) -> usize { + (num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1 + } + + #[inline(always)] + fn data_path(&self) -> PathBuf { + Self::data_path_from(self.path()) + } + #[inline(always)] + fn data_path_from(path: &Path) -> PathBuf { + path.join("data") + } + + #[inline(always)] + pub fn path(&self) -> &Path { + &self.0.path + } + + #[inline] + pub fn weak_clone(&self) -> WeakDatabase { + WeakDatabase(Arc::downgrade(&self.0)) + } + + #[inline] + pub fn name(&self) -> &str { + &self.0.name + } +} + +impl Drop for Database { + fn drop(&mut self) { + if Arc::strong_count(&self.0) == 1 { + let _ = self.sync_bg_tasks(); + } + } +} + +impl fmt::Display for Database { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name()) + } +} + +/// Weak reference to a [`Database`], held by regions to avoid reference cycles. +#[derive(Debug, Clone)] +pub struct WeakDatabase(Weak); + +impl WeakDatabase { + pub fn upgrade(&self) -> Database { + Database( + self.0 + .upgrade() + .expect("Database was dropped while Region still exists"), + ) + } +} diff --git a/crates/rawdb/src/mmap.rs b/crates/rawdb/src/mmap.rs new file mode 100644 index 000000000..198870e4f --- /dev/null +++ b/crates/rawdb/src/mmap.rs @@ -0,0 +1,24 @@ +use std::fs::File; + +use memmap2::{MmapMut, MmapOptions}; + +use crate::Result; + +#[inline] +pub fn create_mmap(file: &File) -> Result { + Ok(unsafe { MmapOptions::new().map_mut(file)? }) +} + +/// Writes `data` at `offset` into the mmap. Panics on out-of-bounds. +#[inline] +pub fn write_to_mmap(mmap: &MmapMut, offset: usize, data: &[u8]) { + let end = offset + .checked_add(data.len()) + .expect("offset + data.len() overflow"); + assert!(end <= mmap.len()); + + unsafe { + let ptr = mmap.as_ptr() as *mut u8; + std::ptr::copy_nonoverlapping(data.as_ptr(), ptr.add(offset), data.len()); + } +} diff --git a/crates/rawdb/src/reader.rs b/crates/rawdb/src/reader.rs new file mode 100644 index 000000000..f1c014727 --- /dev/null +++ b/crates/rawdb/src/reader.rs @@ -0,0 +1,86 @@ +use memmap2::MmapMut; +use parking_lot::RwLockReadGuard; + +use crate::{Database, Region}; + +/// Zero-copy reader with a snapshot of region start/len. +/// +/// Drop as soon as possible — blocks file growth and compaction while held. +#[must_use = "Reader holds locks and should be used for reading"] +pub struct Reader { + // SAFETY: Drop order matters. `mmap` (the lock guard) must drop before `_db` + // (the Arc). Rust drops fields in declaration order, so this is correct. + mmap: RwLockReadGuard<'static, MmapMut>, + start: usize, + len: usize, + _region: Region, + _db: Database, +} + +impl Reader { + #[inline] + pub(crate) fn new(region: &Region) -> Self { + let db = region.db(); + let region = region.clone(); + + let meta = region.meta(); + let start = meta.start(); + let len = meta.len(); + drop(meta); + + // SAFETY: Transmute extends the guard lifetime to 'static. This is safe + // because `_db` (the Arc) outlives `mmap` (the guard) — see struct field order. + let mmap: RwLockReadGuard<'static, MmapMut> = unsafe { std::mem::transmute(db.mmap()) }; + + Self { + _db: db, + _region: region, + start, + len, + mmap, + } + } + + /// # Safety + /// Caller must ensure `offset + len <= self.len()`. + #[inline(always)] + pub fn unchecked_read(&self, offset: usize, len: usize) -> &[u8] { + let start = self.start() + offset; + let end = start + len; + &self.mmap[start..end] + } + + #[inline(always)] + pub fn read(&self, offset: usize, len: usize) -> &[u8] { + assert!(offset + len <= self.len()); + self.unchecked_read(offset, len) + } + + #[inline(always)] + fn start(&self) -> usize { + self.start + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.len + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline(always)] + pub fn read_all(&self) -> &[u8] { + self.read(0, self.len()) + } + + /// Slice from offset to end of mmap (may extend past region boundary). + #[inline(always)] + pub fn prefixed(&self, offset: usize) -> &[u8] { + assert!(offset <= self.len()); + let start = self.start() + offset; + &self.mmap[start..] + } +} diff --git a/crates/rawdb/src/region.rs b/crates/rawdb/src/region.rs new file mode 100644 index 000000000..c79193ce6 --- /dev/null +++ b/crates/rawdb/src/region.rs @@ -0,0 +1,457 @@ +use std::{fs::File, mem, sync::Arc}; + +use log::{debug, trace}; +use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::{Database, Error, Reader, RegionMetadata, Result, WeakDatabase}; + +/// Named, dynamically-sized region within a database. +#[derive(Debug, Clone)] +#[must_use = "Region should be stored to access the data"] +pub struct Region(Arc); + +#[derive(Debug)] +pub(crate) struct RegionInner { + db: WeakDatabase, + index: usize, + meta: RwLock, + /// (min_offset, max_offset) relative to region start. (usize::MAX, 0) = clean. + dirty_bounds: Mutex<(usize, usize)>, +} + +impl Region { + pub(crate) fn new( + db: &Database, + id: String, + index: usize, + start: usize, + len: usize, + reserved: usize, + ) -> Self { + Self(Arc::new(RegionInner { + db: db.weak_clone(), + index, + meta: RwLock::new(RegionMetadata::new(id, start, len, reserved)), + dirty_bounds: Mutex::new((usize::MAX, 0)), + })) + } + + pub(crate) fn from(db: &Database, index: usize, meta: RegionMetadata) -> Self { + Self(Arc::new(RegionInner { + db: db.weak_clone(), + index, + meta: RwLock::new(meta), + dirty_bounds: Mutex::new((usize::MAX, 0)), + })) + } + + #[inline] + pub fn create_reader(&self) -> Reader { + Reader::new(self) + } + + /// Runs `f` with the region's current bytes while holding the mmap read lock. + /// + /// Unlike [`Reader`], this is intended for a single scoped read and does not + /// clone the region or extend a lock guard's lifetime. + #[inline] + pub fn with_read_bytes(&self, f: impl FnOnce(&[u8]) -> R) -> R { + let db = self.db(); + let meta = self.meta(); + let start = meta.start(); + let end = start + meta.len(); + drop(meta); + + let mmap = db.mmap(); + f(&mmap[start..end]) + } + + pub fn open_db_read_only_file(&self) -> Result { + self.db().open_read_only_file() + } + + /// Appends data to the region. Not durable until `flush()`. + #[inline] + pub fn write(&self, data: &[u8]) -> Result<()> { + self.write_with(data, None, false) + } + + /// Writes data at offset within the region. Not durable until `flush()`. + #[inline] + pub fn write_at(&self, data: &[u8], at: usize) -> Result<()> { + self.write_with(data, Some(at), false) + } + + /// Writes (offset, value) pairs directly to the mmap within region bounds. + #[inline] + pub fn batch_write_each( + &self, + iter: impl Iterator, + value_len: usize, + mut write_fn: F, + ) where + F: FnMut(&T, &mut [u8]), + { + let meta = self.meta(); + let region_start = meta.start(); + let region_len = meta.len(); + drop(meta); + + let db = self.db(); + let mmap = db.mmap(); + let ptr = mmap.as_ptr() as *mut u8; + + let mut dirty_start = usize::MAX; + let mut dirty_end = 0usize; + + for (offset, value) in iter { + let end_offset = offset + .checked_add(value_len) + .expect("offset + value_len overflow"); + assert!(end_offset <= region_len); + + let abs_offset = region_start + offset; + let slice = unsafe { std::slice::from_raw_parts_mut(ptr.add(abs_offset), value_len) }; + write_fn(&value, slice); + dirty_start = dirty_start.min(offset); + dirty_end = dirty_end.max(end_offset); + } + + if dirty_start < dirty_end { + let mut bounds = self.0.dirty_bounds.lock(); + bounds.0 = bounds.0.min(dirty_start); + bounds.1 = bounds.1.max(dirty_end); + } + } + + pub fn truncate(&self, from: usize) -> Result<()> { + let len = self.meta().len(); + if from == len { + return Ok(()); + } else if from > len { + return Err(Error::TruncateInvalid { + from, + current_len: len, + }); + } + + let db = self.db(); + // Lock order: regions -> metadata (top-to-bottom) + let regions = db.regions(); + let mut meta = self.meta_mut(); + meta.set_len(from); + meta.write_if_dirty(self.index(), ®ions); + Ok(()) + } + + /// Truncates to `at`, then writes data there. + #[inline] + pub fn truncate_write(&self, at: usize, data: &[u8]) -> Result<()> { + self.write_with(data, Some(at), true) + } + + #[inline] + fn write_with(&self, data: &[u8], at: Option, truncate: bool) -> Result<()> { + let db = self.db(); + let index = self.index(); + let meta = self.meta(); + let start = meta.start(); + let reserved = meta.reserved(); + let len = meta.len(); + drop(meta); + + let data_len = data.len(); + + if let Some(at_val) = at + && at_val > len + { + return Err(Error::WriteOutOfBounds { + position: at_val, + region_len: len, + }); + } + + let write_offset = at.unwrap_or(len); + let new_len = at.map_or(len + data_len, |at| { + let new_len = at + data_len; + if truncate { new_len } else { new_len.max(len) } + }); + let write_start = start + write_offset; + + // --- Fits in reserved space --- + if new_len <= reserved { + db.write(write_start, data); + self.mark_dirty_abs(start, write_start, data_len); + + if new_len != len { + let regions = db.regions(); + let mut meta = self.meta_mut(); + meta.set_len(new_len); + meta.write_if_dirty(index, ®ions); + } + + return Ok(()); + } + + if reserved == 0 { + return Err(Error::InvariantViolation(format!( + "reserved is 0 which would cause infinite loop! start={start}, len={len}, index={index}, new_len={new_len}" + ))); + } + + let mut new_reserved = reserved; + while new_len > new_reserved { + new_reserved = new_reserved + .checked_mul(2) + .ok_or(Error::RegionSizeOverflow { + current: new_reserved, + requested: new_len, + })?; + } + let added_reserve = new_reserved - reserved; + + let copy_len = if truncate { write_offset } else { len }; + + trace!( + "{}: '{}' write_with acquiring layout_mut (need to grow)", + db, + self.meta().id() + ); + let mut layout = db.layout_mut(); + + // --- Extend last region in file --- + if layout.is_last_anything(self) { + let target_len = start + new_reserved; + // Update reserved before dropping layout so Layout::len() is correct. + { + let mut meta = self.meta_mut(); + meta.set_reserved(new_reserved); + } + // Drop layout before set_min_len (needs mmap_mut — would deadlock). + drop(layout); + + if let Err(e) = db.set_min_len(target_len) { + let mut meta = self.meta_mut(); + meta.set_reserved(reserved); + return Err(e); + } + + db.write(write_start, data); + + self.mark_dirty_abs(start, write_start, data_len); + let regions = db.regions(); + let mut meta = self.meta_mut(); + meta.set_len(new_len); + meta.write_if_dirty(index, ®ions); + + return Ok(()); + } + + // --- Expand into adjacent hole --- + let hole_start = start + reserved; + if layout + .get_hole(hole_start) + .is_some_and(|gap| gap >= added_reserve) + { + layout.remove_or_compress_hole(hole_start, added_reserve)?; + let mut meta = self.meta_mut(); + meta.set_reserved(new_reserved); + drop(meta); + drop(layout); + + db.write(write_start, data); + + self.mark_dirty_abs(start, write_start, data_len); + let regions = db.regions(); + let mut meta = self.meta_mut(); + meta.set_len(new_len); + meta.write_if_dirty(index, ®ions); + + return Ok(()); + } + + // --- Relocate to a hole or append at end --- + let new_start = if let Some(hole_start) = layout.find_smallest_adequate_hole(new_reserved) { + debug!( + "{}: '{}' relocating to hole at {} (need {})", + db, + self.meta().id(), + hole_start, + new_reserved + ); + layout.remove_or_compress_hole(hole_start, new_reserved)?; + layout.reserve(hole_start, new_reserved); + drop(layout); + hole_start + } else { + let new_start = layout.len(); + let target_len = new_start + new_reserved; + debug!( + "{}: '{}' allocating at end {} (need {})", + db, + self.meta().id(), + new_start, + new_reserved + ); + // Reserve before dropping layout so other threads see updated len(). + layout.reserve(new_start, new_reserved); + // Drop layout before set_min_len (needs mmap_mut — would deadlock). + drop(layout); + + if let Err(e) = db.set_min_len(target_len) { + let mut layout = db.layout_mut(); + layout.take_reserved(new_start); + return Err(e); + } + new_start + }; + + db.copy(start, new_start, copy_len)?; + db.write(new_start + write_offset, data); + + trace!( + "{}: '{}' write_with re-acquiring layout_mut (after relocation)", + db, + self.meta().id() + ); + let mut layout = db.layout_mut(); + layout.move_region(new_start, self)?; + assert!(layout.take_reserved(new_start) == Some(new_reserved)); + + self.mark_dirty(0, new_len); + let regions = db.regions(); + let mut meta = self.meta_mut(); + meta.set_start(new_start); + meta.set_reserved(new_reserved); + meta.set_len(new_len); + meta.write_if_dirty(index, ®ions); + + Ok(()) + } + + pub fn rename(&self, new_id: &str) -> Result<()> { + let old_id = self.meta().id().to_string(); + let db = self.db(); + debug!("{}: rename '{}' -> '{}'", db, old_id, new_id); + trace!( + "{}: rename '{}' -> '{}' acquiring regions_mut", + db, old_id, new_id + ); + let mut regions = db.regions_mut(); + let mut meta = self.meta_mut(); + let index = self.index(); + regions.rename(&old_id, new_id)?; + meta.set_id(new_id.to_string()); + meta.write_if_dirty(index, ®ions); + Ok(()) + } + + /// Space becomes reusable after the next `flush()`. + pub fn remove(self) -> Result<()> { + let db = self.db(); + let id = self.meta().id().to_string(); + debug!("{}: '{}' remove", db, id); + trace!("{}: '{}' remove acquiring layout_mut", db, id); + // Lock order: layout → regions + let mut layout = db.layout_mut(); + trace!("{}: '{}' remove acquiring regions_mut", db, id); + let mut regions = db.regions_mut(); + trace!("{}: '{}' remove got locks", db, id); + layout.remove_region(&self)?; + regions.remove(&self)?; + Ok(()) + } + + /// Flushes dirty data and metadata to disk. Returns whether anything was flushed. + pub fn flush(&self) -> Result { + let db = self.db(); + let dirty_bounds = self.take_dirty_bounds(); + let regions = db.regions(); + + let data_flushed = if let Some((min, max)) = dirty_bounds { + let region_start = self.meta().start(); + let mmap = db.mmap(); + if let Err(e) = mmap.flush_async_range(region_start + min, max - min) { + drop(mmap); + self.restore_dirty_bounds(min, max); + return Err(e.into()); + } + true + } else { + false + }; + + let meta = self.meta(); + let meta_flushed = meta.flush(self.index(), ®ions)?; + + // Data MUST be durable before metadata — if we crash after metadata sync + // but before data sync, metadata could reference unwritten data. + if data_flushed || meta_flushed { + db.file().sync_data()?; + regions.sync_data()?; + } + + Ok(data_flushed || meta_flushed) + } + + #[inline(always)] + pub fn ptr_eq(&self, other: &Region) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + #[inline(always)] + pub(crate) fn arc(&self) -> &Arc { + &self.0 + } + + #[inline(always)] + pub fn index(&self) -> usize { + self.0.index + } + + #[inline(always)] + pub fn meta(&self) -> RwLockReadGuard<'_, RegionMetadata> { + self.0.meta.read() + } + + #[inline(always)] + pub(crate) fn meta_mut(&self) -> RwLockWriteGuard<'_, RegionMetadata> { + self.0.meta.write() + } + + #[inline(always)] + pub fn db(&self) -> Database { + self.0.db.upgrade() + } + + #[inline] + pub fn mark_dirty(&self, offset: usize, len: usize) { + let end = offset + len; + let mut bounds = self.0.dirty_bounds.lock(); + bounds.0 = bounds.0.min(offset); + bounds.1 = bounds.1.max(end); + } + + #[inline] + fn mark_dirty_abs(&self, region_start: usize, abs_start: usize, len: usize) { + let offset = abs_start - region_start; + self.mark_dirty(offset, len); + } + + #[inline] + pub(crate) fn take_dirty_bounds(&self) -> Option<(usize, usize)> { + let mut bounds = self.0.dirty_bounds.lock(); + if bounds.0 < bounds.1 { + Some(mem::replace(&mut *bounds, (usize::MAX, 0))) + } else { + None + } + } + + #[inline] + pub(crate) fn restore_dirty_bounds(&self, min: usize, max: usize) { + let mut bounds = self.0.dirty_bounds.lock(); + bounds.0 = bounds.0.min(min); + bounds.1 = bounds.1.max(max); + } +} diff --git a/crates/rawdb/src/region_metadata.rs b/crates/rawdb/src/region_metadata.rs new file mode 100644 index 000000000..60c928d48 --- /dev/null +++ b/crates/rawdb/src/region_metadata.rs @@ -0,0 +1,258 @@ +use std::fmt; + +use crate::{Error, GiB, PAGE_SIZE, RegionState, Regions, Result}; + +pub const SIZE_OF_REGION_METADATA: usize = PAGE_SIZE; // 4096 bytes for atomic writes +const SIZE_OF_U64: usize = std::mem::size_of::(); +const MAX_REGION_ID_LEN: usize = 1024; +const MAX_RESERVED_SIZE: usize = 1024 * GiB; // 1 TiB + +/// Serializable metadata for a region (one page, atomic writes). +#[derive(Debug)] +pub struct RegionMetadata { + start: usize, + len: usize, + reserved: usize, + id: String, + state: RegionState, +} + +impl RegionMetadata { + fn validate_id(id: &str) { + assert!(!id.is_empty(), "Region id must not be empty"); + assert!( + id.len() <= MAX_REGION_ID_LEN, + "Region id must be <= {} bytes", + MAX_REGION_ID_LEN + ); + assert!( + !id.chars().any(|c| c.is_control()), + "Region id must not contain control characters" + ); + } + + pub fn new(id: String, start: usize, len: usize, reserved: usize) -> Self { + assert!(start.is_multiple_of(PAGE_SIZE)); + assert!(reserved >= PAGE_SIZE); + assert!(reserved.is_multiple_of(PAGE_SIZE)); + assert!(len <= reserved); + Self::validate_id(&id); + + Self { + id, + len, + reserved, + start, + state: RegionState::new_dirty(), // New region needs write + } + } + + #[inline(always)] + pub fn start(&self) -> usize { + self.start + } + + #[inline] + pub fn set_start(&mut self, start: usize) { + assert!(start.is_multiple_of(PAGE_SIZE)); + Self::update_value_if_different(&mut self.start, start, &self.state) + } + + #[allow(clippy::len_without_is_empty)] + #[inline(always)] + pub fn len(&self) -> usize { + self.len + } + + #[inline] + pub fn set_len(&mut self, len: usize) { + assert!(len <= self.reserved()); + Self::update_value_if_different(&mut self.len, len, &self.state) + } + + #[inline(always)] + pub fn reserved(&self) -> usize { + self.reserved + } + + #[inline(always)] + pub fn id(&self) -> &str { + &self.id + } + + pub fn set_id(&mut self, id: String) { + Self::validate_id(&id); + Self::update_value_if_different(&mut self.id, id, &self.state) + } + + pub fn set_reserved(&mut self, reserved: usize) { + assert!(self.len() <= reserved); + assert!(reserved >= PAGE_SIZE); + assert!(reserved.is_multiple_of(PAGE_SIZE)); + assert!(reserved <= MAX_RESERVED_SIZE); + + Self::update_value_if_different(&mut self.reserved, reserved, &self.state) + } + + #[inline] + fn update_value_if_different(own: &mut T, other: T, state: &RegionState) + where + T: Eq, + { + if own != &other { + *own = other; + state.set_needs_write(); + } + } + + #[inline(always)] + pub fn remaining(&self) -> usize { + self.reserved - self.len + } + + pub(crate) fn write_if_dirty(&self, index: usize, regions: &Regions) { + let state = &self.state; + if state.needs_write() { + regions.write_at(index, &self.to_bytes()); + state.set_needs_flush(); + } + } + + pub(crate) fn flush(&self, index: usize, regions: &Regions) -> Result { + let state = &self.state; + if state.is_clean() { + return Ok(false); + } else if state.needs_write() { + return Err(Error::RegionMetadataUnwritten); + } + // Schedule writeback, then mark clean. Caller ensures durability via sync_data(). + regions + .mmap() + .flush_async_range(index * SIZE_OF_REGION_METADATA, SIZE_OF_REGION_METADATA)?; + state.set_is_clean(); + Ok(true) + } + + #[inline] + pub(crate) fn needs_flush(&self) -> bool { + self.state.needs_flush() + } + + #[inline] + pub(crate) fn mark_clean(&self) { + self.state.set_is_clean(); + } + + fn to_bytes(&self) -> [u8; SIZE_OF_REGION_METADATA] { + let mut pos = 0; + let mut bytes = [0u8; SIZE_OF_REGION_METADATA]; + + bytes[pos..pos + SIZE_OF_U64].copy_from_slice(&(self.start as u64).to_le_bytes()); + pos += SIZE_OF_U64; + + bytes[pos..pos + SIZE_OF_U64].copy_from_slice(&(self.len as u64).to_le_bytes()); + pos += SIZE_OF_U64; + + bytes[pos..pos + SIZE_OF_U64].copy_from_slice(&(self.reserved as u64).to_le_bytes()); + pos += SIZE_OF_U64; + + let id_bytes = self.id.as_bytes(); + let id_len = id_bytes.len(); + bytes[pos..pos + SIZE_OF_U64].copy_from_slice(&(id_len as u64).to_le_bytes()); + pos += SIZE_OF_U64; + + bytes[pos..pos + id_len].copy_from_slice(id_bytes); + + bytes + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() != SIZE_OF_REGION_METADATA { + return Err(Error::InvalidMetadataSize { + expected: SIZE_OF_REGION_METADATA, + actual: bytes.len(), + }); + } + + let start = u64::from_le_bytes(bytes[0..8].try_into().unwrap()) as usize; + let len = u64::from_le_bytes(bytes[8..16].try_into().unwrap()) as usize; + let reserved = u64::from_le_bytes(bytes[16..24].try_into().unwrap()) as usize; + let id_len = u64::from_le_bytes(bytes[24..32].try_into().unwrap()) as usize; + + if start == 0 && len == 0 && reserved == 0 && id_len == 0 { + return Err(Error::EmptyMetadata); + } + + if id_len > MAX_REGION_ID_LEN { + return Err(Error::CorruptedMetadata(format!( + "id_len {} exceeds maximum {}", + id_len, MAX_REGION_ID_LEN + ))); + } + + if 32 + id_len > SIZE_OF_REGION_METADATA { + return Err(Error::CorruptedMetadata(format!( + "id_len {} would exceed metadata size", + id_len + ))); + } + + let id = String::from_utf8(bytes[32..32 + id_len].to_vec()) + .map_err(|_| Error::InvalidRegionId)?; + + if !start.is_multiple_of(PAGE_SIZE) { + return Err(Error::CorruptedMetadata(format!( + "start {} is not page-aligned", + start + ))); + } + if reserved < PAGE_SIZE { + return Err(Error::CorruptedMetadata(format!( + "reserved {} is less than PAGE_SIZE", + reserved + ))); + } + if !reserved.is_multiple_of(PAGE_SIZE) { + return Err(Error::CorruptedMetadata(format!( + "reserved {} is not page-aligned", + reserved + ))); + } + if len > reserved { + return Err(Error::CorruptedMetadata(format!( + "len {} exceeds reserved {}", + len, reserved + ))); + } + + Ok(Self { + id, + start, + len, + reserved, + state: RegionState::new_clean(), // Loaded from disk + }) + } +} + +impl Clone for RegionMetadata { + fn clone(&self) -> Self { + Self { + start: self.start, + len: self.len, + reserved: self.reserved, + id: self.id.clone(), + state: RegionState::new_clean(), + } + } +} + +impl fmt::Display for RegionMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "'{}' (start={}, len={}, reserved={})", + self.id, self.start, self.len, self.reserved + ) + } +} diff --git a/crates/rawdb/src/region_state.rs b/crates/rawdb/src/region_state.rs new file mode 100644 index 000000000..abc23bbed --- /dev/null +++ b/crates/rawdb/src/region_state.rs @@ -0,0 +1,56 @@ +use std::sync::atomic::{AtomicU8, Ordering}; + +/// State transitions: NEEDS_WRITE → NEEDS_FLUSH → IS_CLEAN. +#[derive(Debug)] +pub struct RegionState(AtomicU8); + +impl RegionState { + pub const IS_CLEAN: u8 = 0; + pub const NEEDS_FLUSH: u8 = 1; + pub const NEEDS_WRITE: u8 = 2; + + #[inline(always)] + pub fn new_dirty() -> Self { + Self(AtomicU8::new(Self::NEEDS_WRITE)) + } + + #[inline(always)] + pub fn new_clean() -> Self { + Self(AtomicU8::new(Self::IS_CLEAN)) + } + + #[inline(always)] + fn load(&self) -> u8 { + self.0.load(Ordering::Acquire) + } + + #[inline(always)] + pub fn is_clean(&self) -> bool { + self.load() == Self::IS_CLEAN + } + + #[inline(always)] + pub fn set_is_clean(&self) { + self.0.store(Self::IS_CLEAN, Ordering::Release); + } + + #[inline(always)] + pub fn needs_flush(&self) -> bool { + self.load() == Self::NEEDS_FLUSH + } + + #[inline(always)] + pub fn set_needs_flush(&self) { + self.0.store(Self::NEEDS_FLUSH, Ordering::Release); + } + + #[inline(always)] + pub fn needs_write(&self) -> bool { + self.load() == Self::NEEDS_WRITE + } + + #[inline(always)] + pub fn set_needs_write(&self) { + self.0.store(Self::NEEDS_WRITE, Ordering::Release); + } +} diff --git a/crates/rawdb/src/regions.rs b/crates/rawdb/src/regions.rs new file mode 100644 index 000000000..39df79122 --- /dev/null +++ b/crates/rawdb/src/regions.rs @@ -0,0 +1,212 @@ +use std::{ + collections::HashMap, + fs::{self, File, OpenOptions}, + path::Path, + sync::Arc, +}; + +use log::debug; +use memmap2::MmapMut; + +use crate::{ + Database, Error, PAGE_SIZE, RegionMetadata, Result, SIZE_OF_REGION_METADATA, create_mmap, + region::Region, write_to_mmap, +}; + +#[derive(Debug)] +pub struct Regions { + id_to_index: HashMap, + index_to_region: Vec>, + file: File, + mmap: MmapMut, +} + +impl Regions { + pub fn open(parent: &Path) -> Result { + fs::create_dir_all(parent)?; + + let file = OpenOptions::new() + .read(true) + .create(true) + .write(true) + .truncate(false) + .open(parent.join("regions"))?; + file.try_lock()?; + + let mmap = create_mmap(&file)?; + + Ok(Self { + id_to_index: HashMap::new(), + index_to_region: vec![], + file, + mmap, + }) + } + + fn file_len(&self) -> Result { + Ok(self.file.metadata()?.len() as usize) + } + + pub(crate) fn fill(&mut self, db: &Database) -> Result<()> { + let file_len = self.file_len()?; + + if file_len % SIZE_OF_REGION_METADATA != 0 { + return Err(Error::CorruptedMetadata(format!( + "regions file size {} is not a multiple of {}", + file_len, SIZE_OF_REGION_METADATA + ))); + } + + let num_slots = file_len / SIZE_OF_REGION_METADATA; + + self.index_to_region + .resize_with(num_slots, Default::default); + + for index in 0..num_slots { + let start = index * SIZE_OF_REGION_METADATA; + let bytes = &self.mmap[start..start + SIZE_OF_REGION_METADATA]; + + let Ok(meta) = RegionMetadata::from_bytes(bytes) else { + continue; + }; + + self.id_to_index.insert(meta.id().to_string(), index); + self.index_to_region[index] = Some(Region::from(db, index, meta)); + } + + Ok(()) + } + + pub(crate) fn set_min_len(&mut self, len: usize) -> Result<()> { + let file_len = self.file_len()?; + if file_len < len { + self.file.set_len(len as u64)?; + self.mmap = create_mmap(&self.file)?; + } + Ok(()) + } + + pub(crate) fn create(&mut self, db: &Database, id: String, start: usize) -> Result { + let index = self + .index_to_region + .iter() + .enumerate() + .find(|(_, opt)| opt.is_none()) + .map(|(index, _)| index) + .unwrap_or_else(|| self.index_to_region.len()); + + let region = Region::new(db, id.clone(), index, start, 0, PAGE_SIZE); + + self.set_min_len((index + 1) * SIZE_OF_REGION_METADATA)?; + + let region_opt = Some(region.clone()); + if index < self.index_to_region.len() { + self.index_to_region[index] = region_opt + } else { + self.index_to_region.push(region_opt); + } + + if self.id_to_index.insert(id, index).is_some() { + return Err(Error::RegionAlreadyExists); + } + + Ok(region) + } + + #[inline] + pub fn get_from_index(&self, index: usize) -> Option<&Region> { + self.index_to_region.get(index).and_then(Option::as_ref) + } + + #[inline] + pub fn get_from_id(&self, id: &str) -> Option<&Region> { + self.id_to_index + .get(id) + .and_then(|&index| self.get_from_index(index)) + } + + pub(crate) fn rename(&mut self, old_id: &str, new_id: &str) -> Result<()> { + let index = self + .id_to_index + .get(old_id) + .copied() + .ok_or(Error::RegionNotFound)?; + + if self.id_to_index.contains_key(new_id) { + return Err(Error::RegionAlreadyExists); + } + + self.id_to_index.remove(old_id); + self.id_to_index.insert(new_id.to_string(), index); + + Ok(()) + } + + pub(crate) fn remove(&mut self, region: &Region) -> Result<()> { + // Expected 2: one from caller, one from self.index_to_region. + let ref_count = Arc::strong_count(region.arc()); + debug!( + "regions.remove '{}': arc count = {} (expected <= 2)", + region.meta().id(), + ref_count + ); + if ref_count > 2 { + return Err(Error::RegionStillReferenced { + id: region.meta().id().to_string(), + ref_count, + }); + } + + if self + .index_to_region + .get_mut(region.index()) + .and_then(Option::take) + .is_none() + { + return Err(Error::RegionNotFound); + } + + self.id_to_index.remove(region.meta().id()); + + self.write_at(region.index(), &[0u8; SIZE_OF_REGION_METADATA]); + + Ok(()) + } + + /// Schedules metadata writeback. Caller must follow with `sync_data()`. + pub(crate) fn flush(&self) -> Result<()> { + self.mmap.flush_async()?; + Ok(()) + } + + pub(crate) fn sync_data(&self) -> Result<()> { + self.file.sync_data()?; + Ok(()) + } + + pub(crate) fn write_at(&self, index: usize, data: &[u8]) { + debug_assert_eq!(data.len(), SIZE_OF_REGION_METADATA); + let offset = index * SIZE_OF_REGION_METADATA; + write_to_mmap(&self.mmap, offset, data); + } + + #[inline] + pub fn index_to_region(&self) -> &[Option] { + &self.index_to_region + } + + #[inline] + pub fn id_to_index(&self) -> &HashMap { + &self.id_to_index + } + + #[inline] + pub fn len(&self) -> usize { + self.id_to_index.len() + } + + #[inline] + pub(crate) fn mmap(&self) -> &MmapMut { + &self.mmap + } +} diff --git a/crates/rawdb/tests/mod.rs b/crates/rawdb/tests/mod.rs new file mode 100644 index 000000000..d9d7cb0ea --- /dev/null +++ b/crates/rawdb/tests/mod.rs @@ -0,0 +1,2077 @@ +use rawdb::{Database, PAGE_SIZE, Result}; +use std::sync::Arc; +use std::thread; +use tempfile::TempDir; + +/// Helper to create a temporary test database +fn setup_test_db() -> Result<(Database, TempDir)> { + let temp_dir = TempDir::new()?; + let db = Database::open(temp_dir.path())?; + Ok((db, temp_dir)) +} + +#[test] +fn test_database_creation() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Database should start empty + assert_eq!(db.regions().index_to_region().len(), 0); + assert_eq!(db.layout().start_to_region().len(), 0); + assert_eq!(db.layout().start_to_hole().len(), 0); + + Ok(()) +} + +#[test] +fn test_create_single_region() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test_region")?; + + // Verify region properties + let meta = region.meta(); + assert_eq!(meta.start(), 0); + assert_eq!(meta.len(), 0); + assert_eq!(meta.reserved(), PAGE_SIZE); + drop(meta); + + // Verify it's tracked in regions + let regions = db.regions(); + assert_eq!(regions.index_to_region().len(), 1); + assert!(regions.get_from_id("test_region").is_some()); + + // Verify it's tracked in layout + let layout = db.layout(); + assert_eq!(layout.start_to_region().len(), 1); + assert!(layout.start_to_hole().is_empty()); + + Ok(()) +} + +#[test] +fn test_create_region_idempotent() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region1 = db.create_region_if_needed("test")?; + let region2 = db.create_region_if_needed("test")?; + + // Should return same region + assert_eq!(region1.index(), region2.index()); + assert_eq!(db.regions().index_to_region().len(), 1); + + Ok(()) +} + +#[test] +fn test_write_to_region_within_reserved() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + let data = b"Hello, World!"; + + region.write(data)?; + + // Verify data was written + let meta = region.meta(); + assert_eq!(meta.len(), data.len()); + assert_eq!(meta.reserved(), PAGE_SIZE); + let start = meta.start(); + drop(meta); + + let mmap = db.mmap(); + assert_eq!(&mmap[start..start + data.len()], data); + + Ok(()) +} + +#[test] +fn test_write_append() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + region.write(b"Hello")?; + region.write(b", World!")?; + + let meta = region.meta(); + assert_eq!(meta.len(), 13); + let start = meta.start(); + drop(meta); + + let mmap = db.mmap(); + assert_eq!(&mmap[start..(start + 13)], b"Hello, World!"); + + Ok(()) +} + +#[test] +fn test_write_at_position() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + region.write(b"Hello, World!")?; + region.write_at(b"Rust!", 7)?; + + let meta = region.meta(); + let start = meta.start(); + drop(meta); + + let mmap = db.mmap(); + assert_eq!(&mmap[start..(start + 13)], b"Hello, Rust!!"); + + Ok(()) +} + +#[test] +fn test_write_exceeds_reserved() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + // Write more than PAGE_SIZE to trigger expansion + let large_data = vec![1u8; PAGE_SIZE + 100]; + region.write(&large_data)?; + + let meta = region.meta(); + assert_eq!(meta.len(), large_data.len()); + assert!(meta.reserved() >= PAGE_SIZE * 2); + + Ok(()) +} + +#[test] +fn test_truncate_region() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + region.write(b"Hello, World!")?; + + let meta_before = region.meta(); + assert_eq!(meta_before.len(), 13); + drop(meta_before); + + region.truncate(5)?; + + let meta_after = region.meta(); + assert_eq!(meta_after.len(), 5); + + Ok(()) +} + +#[test] +fn test_truncate_errors() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + region.write(b"Hello")?; + + // Truncating beyond length should error + let result = region.truncate(10); + assert!(result.is_err()); + + // Truncating to same length should be OK + let result = region.truncate(5); + assert!(result.is_ok()); + + Ok(()) +} + +#[test] +fn test_remove_region() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + let index = region.index(); + + region.write(b"Hello")?; + + // Remove region + region.remove()?; + + // Verify removal + let regions = db.regions(); + assert!(regions.get_from_id("test").is_none()); + assert!(regions.get_from_index(index).is_none()); + + db.flush()?; // Make hole available + + // Layout should have a hole now + let layout = db.layout(); + assert_eq!(layout.start_to_region().len(), 0); + assert_eq!(layout.start_to_hole().len(), 1); + + Ok(()) +} + +#[test] +fn test_multiple_regions() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region1 = db.create_region_if_needed("region1")?; + let region2 = db.create_region_if_needed("region2")?; + let region3 = db.create_region_if_needed("region3")?; + + // Write different data to each + region1.write(b"First")?; + region2.write(b"Second")?; + region3.write(b"Third")?; + + // Verify all exist + assert_eq!(db.regions().index_to_region().len(), 3); + assert_eq!(db.layout().start_to_region().len(), 3); + + // Verify data integrity + let mmap = db.mmap(); + + let meta1 = region1.meta(); + assert_eq!(&mmap[meta1.start()..(meta1.start() + 5)], b"First"); + drop(meta1); + + let meta2 = region2.meta(); + assert_eq!(&mmap[meta2.start()..(meta2.start() + 6)], b"Second"); + drop(meta2); + + let meta3 = region3.meta(); + assert_eq!(&mmap[meta3.start()..(meta3.start() + 5)], b"Third"); + + Ok(()) +} + +#[test] +fn test_region_reuse_after_removal() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region1 = db.create_region_if_needed("region1")?; + let _region2 = db.create_region_if_needed("region2")?; + let index1 = region1.index(); + + // Remove first region + region1.remove()?; + + // Create a new region - should reuse the slot + let region3 = db.create_region_if_needed("region3")?; + assert_eq!(region3.index(), index1); + + // Verify only 2 regions exist + let regions = db.regions(); + assert_eq!(regions.id_to_index().len(), 2); + assert!(regions.get_from_id("region1").is_none()); + assert!(regions.get_from_id("region2").is_some()); + assert!(regions.get_from_id("region3").is_some()); + + Ok(()) +} + +#[test] +fn test_hole_filling() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let _region1 = db.create_region_if_needed("region1")?; + let region2 = db.create_region_if_needed("region2")?; + let _region3 = db.create_region_if_needed("region3")?; + + // Remove middle region to create a hole + region2.remove()?; + db.flush()?; // Make hole available for reuse + + let layout = db.layout(); + assert_eq!(layout.start_to_hole().len(), 1); + drop(layout); + + // Create new region - should fill the hole + let _region4 = db.create_region_if_needed("region4")?; + + let layout = db.layout(); + // Hole should be gone since new region takes PAGE_SIZE which fills it exactly + assert_eq!(layout.start_to_hole().len(), 0); + + Ok(()) +} + +#[test] +fn test_persistence() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path(); + dbg!(&path); + + // Create and populate database + { + let db = Database::open(path)?; + let region = db.create_region_if_needed("persistent")?; + region.write(b"Persisted data")?; + db.flush()?; + } + + // Reopen and verify + { + let db = Database::open(path)?; + let regions = db.regions(); + let region = regions + .get_from_id("persistent") + .expect("Region should persist"); + + let meta = region.meta(); + assert_eq!(meta.len(), 14); + let start = meta.start(); + drop(meta); + + let mmap = db.mmap(); + assert_eq!(&mmap[start..(start + 14)], b"Persisted data"); + } + + Ok(()) +} + +#[test] +fn test_reader() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + region.write(b"Hello, World!")?; + + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b"Hello, World!"); + assert_eq!(reader.read(0, 5), b"Hello"); + assert_eq!(reader.read(7, 5), b"World"); + + Ok(()) +} + +#[test] +fn test_retain_regions() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let _ = db.create_region_if_needed("keep1")?; + let _ = db.create_region_if_needed("remove1")?; + let _ = db.create_region_if_needed("keep2")?; + let _ = db.create_region_if_needed("remove2")?; + + let mut keep_set = std::collections::HashSet::new(); + keep_set.insert("keep1".to_string()); + keep_set.insert("keep2".to_string()); + + db.retain_regions(keep_set)?; + + let regions = db.regions(); + assert_eq!(regions.id_to_index().len(), 2); + assert!(regions.get_from_id("keep1").is_some()); + assert!(regions.get_from_id("keep2").is_some()); + assert!(regions.get_from_id("remove1").is_none()); + assert!(regions.get_from_id("remove2").is_none()); + + Ok(()) +} + +#[test] +fn test_region_defragmentation() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region1 = db.create_region_if_needed("region1")?; + let region2 = db.create_region_if_needed("region2")?; + + dbg!(0); + + // Write small data first + region1.write(b"small")?; + + dbg!(1); + // Write large data to region1 - should move it to end + let large_data = vec![1u8; PAGE_SIZE * 2]; + region1.write(&large_data)?; + db.flush()?; // Make hole available + + dbg!(2); + // region1 should have moved, leaving a hole + let layout = db.layout(); + assert!(layout.start_to_hole().len() == 1); + + dbg!(3); + // region2 should still be at its original position + let meta2 = region2.meta(); + assert_eq!(meta2.start(), PAGE_SIZE); + dbg!(4); + + Ok(()) +} + +#[test] +fn test_concurrent_region_creation() -> Result<()> { + let temp = TempDir::new()?; + let db = Arc::new(Database::open(temp.path())?); + + let handles: Vec<_> = (0..10) + .map(|i| { + let db = Arc::clone(&db); + thread::spawn(move || { + let region_name = format!("region_{}", i); + db.create_region_if_needed(®ion_name) + }) + }) + .collect(); + + // Wait for all threads + for handle in handles { + let _ = handle.join().unwrap()?; + } + + // Verify all regions created + let regions = db.regions(); + assert_eq!(regions.id_to_index().len(), 10); + + Ok(()) +} + +#[test] +fn test_set_min_regions() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + db.set_min_regions(100)?; + + // File should be large enough for 100 regions + let file_len = db.file_len(); + assert!(file_len >= 100 * PAGE_SIZE); + + Ok(()) +} + +#[test] +fn test_large_write() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("large")?; + + // Write 1MB of data + let large_data = vec![42u8; 1024 * 1024]; + region.write(&large_data)?; + + let meta = region.meta(); + assert_eq!(meta.len(), large_data.len()); + let start = meta.start(); + drop(meta); + + // Verify data + let mmap = db.mmap(); + assert_eq!(&mmap[start..(start + large_data.len())], &large_data[..]); + + Ok(()) +} + +#[test] +fn test_truncate_write() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + region.write(b"Hello, World!")?; + + let meta_before = region.meta(); + assert_eq!(meta_before.len(), 13); + drop(meta_before); + + // Truncate write - should set length to exactly the written data + region.truncate_write(7, b"Rust")?; + + let meta_after = region.meta(); + assert_eq!(meta_after.len(), 11); // 7 + 4 + let start = meta_after.start(); + drop(meta_after); + + let mmap = db.mmap(); + assert_eq!(&mmap[start..(start + 11)], b"Hello, Rust"); + + Ok(()) +} + +#[test] +fn test_punch_holes() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + // Write large data then truncate + let large_data = vec![1u8; PAGE_SIZE * 2]; + region.write(&large_data)?; + region.truncate(100)?; + + // Flush and punch holes + db.compact()?; + + // Should still be able to read the data + let meta = region.meta(); + assert_eq!(meta.len(), 100); + + Ok(()) +} + +#[test] +fn test_write_at_invalid_position() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + region.write(b"Hello")?; + + // Writing beyond length should fail + let result = region.write_at(b"World", 10); + assert!(result.is_err()); + + Ok(()) +} + +#[test] +fn test_empty_region_operations() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("empty")?; + + // Reading empty region + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b""); + drop(reader); + + // Truncating empty region to 0 should work + region.truncate(0)?; + + let meta = region.meta(); + assert_eq!(meta.len(), 0); + + Ok(()) +} + +#[test] +fn test_region_metadata_updates() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + + // Initial state + { + let meta = region.meta(); + assert_eq!(meta.start(), 0); + assert_eq!(meta.len(), 0); + assert_eq!(meta.reserved(), PAGE_SIZE); + } + + // After first write + region.write(b"Hello")?; + { + let meta = region.meta(); + assert_eq!(meta.len(), 5); + assert_eq!(meta.reserved(), PAGE_SIZE); + } + + // After expansion + let large = vec![1u8; PAGE_SIZE * 3]; + region.write(&large)?; + { + let meta = region.meta(); + assert_eq!(meta.len(), 5 + large.len()); + assert!(meta.reserved() >= PAGE_SIZE * 4); + } + + Ok(()) +} + +// ============================================================================ +// Complex Integration Tests +// ============================================================================ + +#[test] +fn test_complex_region_lifecycle() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create multiple regions + let r1 = db.create_region_if_needed("region1")?; + let r2 = db.create_region_if_needed("region2")?; + let r3 = db.create_region_if_needed("region3")?; + + // Write to all regions + r1.write(b"Data for region 1")?; + r2.write(b"Data for region 2")?; + r3.write(b"Data for region 3")?; + + // Remove middle region + r2.remove()?; + db.flush()?; // Make hole available + + // Verify hole exists + { + let layout = db.layout(); + assert_eq!(layout.start_to_hole().len(), 1); + } + + // Create new region that should reuse the hole + let r4 = db.create_region_if_needed("region4")?; + r4.write(b"Fills the hole")?; + + // Verify hole was filled + { + let layout = db.layout(); + assert_eq!(layout.start_to_hole().len(), 0); + } + + // Write large data to trigger region movement (overwrite from start) + let large = vec![42u8; PAGE_SIZE * 3]; + r4.write_at(&large, 0)?; + db.flush()?; // Make hole available + + // Verify r4 moved and created a hole + { + let layout = db.layout(); + assert!(!layout.start_to_hole().is_empty()); + } + + // Verify all data is still correct + { + let reader = r1.create_reader(); + assert_eq!(reader.read_all(), b"Data for region 1"); + drop(reader); + } + + { + let reader = r3.create_reader(); + assert_eq!(reader.read_all(), b"Data for region 3"); + drop(reader); + } + + { + let reader = r4.create_reader(); + assert_eq!(reader.read_all(), &large[..]); + drop(reader); + } + + Ok(()) +} + +#[test] +fn test_many_small_regions() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create 50 small regions + let mut regions = Vec::new(); + for i in 0..50 { + let name = format!("region_{}", i); + let region = db.create_region_if_needed(&name)?; + let data = format!("Data for region {}", i); + region.write(data.as_bytes())?; + regions.push(Some(region)); + } + + // Verify all regions + for (i, region_opt) in regions.iter().enumerate() { + let region = region_opt.as_ref().unwrap(); + let reader = region.create_reader(); + let expected = format!("Data for region {}", i); + assert_eq!(reader.read_all(), expected.as_bytes()); + drop(reader); + } + + // Remove every other region + for i in (0..50).step_by(2) { + let region = regions[i].take().unwrap(); + region.remove()?; + } + + // Verify remaining regions + for i in (1..50).step_by(2) { + let region = regions[i].as_ref().unwrap(); + let reader = region.create_reader(); + let expected = format!("Data for region {}", i); + assert_eq!(reader.read_all(), expected.as_bytes()); + drop(reader); + } + + Ok(()) +} + +#[test] +fn test_interleaved_operations() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let r1 = db.create_region_if_needed("r1")?; + let r2 = db.create_region_if_needed("r2")?; + let r3 = db.create_region_if_needed("r3")?; + + // Interleave writes + r1.write(b"Start1")?; + r2.write(b"Start2")?; + r3.write(b"Start3")?; + + r1.write(b" More1")?; + r2.write(b" More2")?; + + // Truncate one + r3.truncate(3)?; + + // Continue writing + r1.write(b" End1")?; + r2.write_at(b"X", 0)?; + + // Verify results + { + let reader = r1.create_reader(); + assert_eq!(reader.read_all(), b"Start1 More1 End1"); + drop(reader); + } + + { + let reader = r2.create_reader(); + assert_eq!(reader.read_all(), b"Xtart2 More2"); + drop(reader); + } + + { + let meta = r3.meta(); + assert_eq!(meta.len(), 3); + } + + Ok(()) +} + +#[test] +fn test_persistence_with_holes() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path(); + + // Create database with regions and holes + { + let db = Database::open(path)?; + + let r1 = db.create_region_if_needed("keep1")?; + let r2 = db.create_region_if_needed("remove")?; + let r3 = db.create_region_if_needed("keep2")?; + + r1.write(b"Keep this 1")?; + r2.write(b"Remove this")?; + r3.write(b"Keep this 2")?; + + r2.remove()?; + db.flush()?; + } + + // Reopen and verify + { + let db = Database::open(path)?; + + let regions = db.regions(); + assert!(regions.get_from_id("keep1").is_some()); + assert!(regions.get_from_id("remove").is_none()); + assert!(regions.get_from_id("keep2").is_some()); + + let r1 = regions.get_from_id("keep1").unwrap(); + let r3 = regions.get_from_id("keep2").unwrap(); + + let reader1 = r1.create_reader(); + assert_eq!(reader1.read_all(), b"Keep this 1"); + drop(reader1); + + let reader3 = r3.create_reader(); + assert_eq!(reader3.read_all(), b"Keep this 2"); + drop(reader3); + + // Verify hole still exists + let layout = db.layout(); + assert!(!layout.start_to_hole().is_empty()); + } + + Ok(()) +} + +#[test] +fn test_region_growth_patterns() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("growing")?; + + // Grow gradually + for i in 0..10 { + let data = vec![i as u8; 1000]; + region.write(&data)?; + } + + let meta = region.meta(); + assert_eq!(meta.len(), 10_000); + + // Verify all data + let reader = region.create_reader(); + let all_data = reader.read_all(); + for i in 0..10 { + let chunk = &all_data[i * 1000..(i + 1) * 1000]; + assert!(chunk.iter().all(|&b| b == i as u8)); + } + + Ok(()) +} + +#[test] +fn test_write_at_boundary_conditions() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("boundary")?; + + // Write at start + region.write(b"0123456789")?; + + // Write at exact length boundary + region.write_at(b"ABC", 10)?; + + // Write at position 0 + region.write_at(b"X", 0)?; + + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b"X123456789ABC"); + + Ok(()) +} + +#[test] +fn test_multiple_flushes() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path(); + + { + let db = Database::open(path)?; + let r = db.create_region_if_needed("test")?; + + r.write(b"Version 1")?; + db.flush()?; + + r.write(b" Version 2")?; + db.flush()?; + + r.write(b" Version 3")?; + db.flush()?; + } + + { + let db = Database::open(path)?; + let regions = db.regions(); + let r = regions.get_from_id("test").unwrap(); + + let reader = r.create_reader(); + assert_eq!(reader.read_all(), b"Version 1 Version 2 Version 3"); + } + + Ok(()) +} + +#[test] +fn test_hole_coalescing() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create 5 regions + let mut regions: Vec<_> = (0..5) + .map(|i| db.create_region_if_needed(&format!("r{}", i))) + .collect::>>()? + .into_iter() + .map(Some) + .collect(); + + // Write small data to each + for r in ®ions { + r.as_ref().unwrap().write(b"data")?; + } + + // Remove regions 1, 2, 3 to create adjacent holes + regions[1].take().unwrap().remove()?; + regions[2].take().unwrap().remove()?; + regions[3].take().unwrap().remove()?; + db.flush()?; // Make holes available and coalesce + + // Check that holes were coalesced + let layout = db.layout(); + // Should have 1 large hole, not 3 separate ones + let holes = layout.start_to_hole(); + assert_eq!(holes.len(), 1); + + // The single hole should span all 3 removed regions + let hole_size = holes.values().next().unwrap(); + assert_eq!(*hole_size, PAGE_SIZE * 3); + + Ok(()) +} + +#[test] +fn test_stress_region_creation_and_removal() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create and remove regions in a cycle + for cycle in 0..5 { + // Create 20 regions + let regions: Vec<_> = (0..20) + .map(|i| { + let name = format!("cycle_{}_region_{}", cycle, i); + db.create_region_if_needed(&name) + }) + .collect::>>()?; + + // Write to each + for (i, r) in regions.iter().enumerate() { + let data = format!("Cycle {} Region {}", cycle, i); + r.write(data.as_bytes())?; + } + + // Verify + for (i, r) in regions.iter().enumerate() { + let reader = r.create_reader(); + let expected = format!("Cycle {} Region {}", cycle, i); + assert_eq!(reader.read_all(), expected.as_bytes()); + drop(reader); + } + + // Remove all + for r in regions { + r.remove()?; + } + + // Verify all gone + let reg = db.regions(); + assert_eq!(reg.id_to_index().len(), 0); + } + + Ok(()) +} + +#[test] +fn test_mixed_size_writes() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("mixed")?; + + // Write various sizes + region.write(b"tiny")?; + region.write(&[1u8; 100])?; + region.write(&[2u8; 1000])?; + region.write(&[3u8; 10000])?; + + let meta = region.meta(); + assert_eq!(meta.len(), 4 + 100 + 1000 + 10000); + + // Verify each section + let reader = region.create_reader(); + assert_eq!(&reader.read(0, 4), b"tiny"); + assert!(reader.read(4, 100).iter().all(|&b| b == 1)); + assert!(reader.read(104, 1000).iter().all(|&b| b == 2)); + assert!(reader.read(1104, 10000).iter().all(|&b| b == 3)); + + Ok(()) +} + +// ============================================================================ +// Concurrent Operations Tests +// ============================================================================ + +#[test] +fn test_concurrent_writes_to_different_regions() -> Result<()> { + let temp = TempDir::new()?; + let db = Arc::new(Database::open(temp.path())?); + + // Create regions upfront + let regions: Vec<_> = (0..10) + .map(|i| db.create_region_if_needed(&format!("region_{}", i))) + .collect::>>()?; + + // Write to different regions concurrently + let handles: Vec<_> = regions + .into_iter() + .enumerate() + .map(|(i, region)| { + thread::spawn(move || { + let data = vec![i as u8; 1000]; + region.write(&data) + }) + }) + .collect(); + + // Wait for all writes + for handle in handles { + handle.join().unwrap()?; + } + + // Verify all data + for i in 0..10 { + let regions = db.regions(); + let region = regions.get_from_id(&format!("region_{}", i)).unwrap(); + let reader = region.create_reader(); + let data = reader.read_all(); + assert_eq!(data.len(), 1000); + assert!(data.iter().all(|&b| b == i as u8)); + } + + Ok(()) +} + +#[test] +fn test_concurrent_reads() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let db = Arc::new(db); + + let region = db.create_region_if_needed("shared")?; + let data = b"Shared data for concurrent reads"; + region.write(data)?; + + // Multiple threads reading simultaneously + let handles: Vec<_> = (0..20) + .map(|_| { + let db = Arc::clone(&db); + thread::spawn(move || { + let regions = db.regions(); + let region = regions.get_from_id("shared").unwrap(); + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b"Shared data for concurrent reads"); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + Ok(()) +} + +// ============================================================================ +// Reader Edge Cases +// ============================================================================ + +#[test] +fn test_reader_prefixed() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + region.write(b"0123456789ABCDEF")?; + + let reader = region.create_reader(); + + // Test prefixed reads + let prefixed = reader.prefixed(5); + assert!(prefixed.starts_with(b"56789ABCDEF")); + + let prefixed_at_start = reader.prefixed(0); + assert!(prefixed_at_start.starts_with(b"0123456789ABCDEF")); + + Ok(()) +} + +#[test] +fn test_reader_unchecked_read() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + region.write(b"Hello World")?; + + let reader = region.create_reader(); + + // Unchecked read within bounds should work + let data = reader.unchecked_read(0, 5); + assert_eq!(data, b"Hello"); + + let data = reader.unchecked_read(6, 5); + assert_eq!(data, b"World"); + + Ok(()) +} + +#[test] +#[should_panic] +fn test_reader_bounds_check() { + let (db, _temp) = setup_test_db().unwrap(); + + let region = db.create_region_if_needed("test").unwrap(); + region.write(b"Short").unwrap(); + + let reader = region.create_reader(); + + // This should panic due to bounds check + let _ = reader.read(0, 100); +} + +// Test that Reader is self-contained and safe to use after dropping original references +#[test] +fn test_reader_outlives_region_variable() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let reader = { + let region = db.create_region_if_needed("test")?; + region.write(b"Hello from dropped region")?; + region.create_reader() + // region is dropped here, but reader should still be valid + }; + + // Reader should still work because it holds its own Arc references + assert_eq!(reader.read_all(), b"Hello from dropped region"); + assert_eq!(reader.read(0, 5), b"Hello"); + + Ok(()) +} + +#[test] +fn test_reader_outlives_database_variable() -> Result<()> { + let (_temp, reader) = { + let (db, temp) = setup_test_db()?; + let region = db.create_region_if_needed("test")?; + region.write(b"Persisted data")?; + let reader = region.create_reader(); + // Both db and region go out of scope, but reader holds Arc clones + (temp, reader) + }; + + // Reader should still work + assert_eq!(reader.read_all(), b"Persisted data"); + + Ok(()) +} + +#[test] +fn test_reader_can_be_stored_in_struct() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + struct DataHolder { + reader: rawdb::Reader, + } + + let region = db.create_region_if_needed("test")?; + region.write(b"Stored in struct")?; + + let holder = DataHolder { + reader: region.create_reader(), + }; + + // Can use reader through struct + assert_eq!(holder.reader.read_all(), b"Stored in struct"); + + Ok(()) +} + +#[test] +fn test_multiple_readers_from_same_region() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("test")?; + region.write(b"Shared data")?; + + // Create multiple readers + let reader1 = region.create_reader(); + let reader2 = region.create_reader(); + let reader3 = region.create_reader(); + + // All should read the same data + assert_eq!(reader1.read_all(), b"Shared data"); + assert_eq!(reader2.read_all(), b"Shared data"); + assert_eq!(reader3.read_all(), b"Shared data"); + + // Drop in different order + drop(reader2); + assert_eq!(reader1.read_all(), b"Shared data"); + assert_eq!(reader3.read_all(), b"Shared data"); + + Ok(()) +} + +// ============================================================================ +// Extreme Cases +// ============================================================================ + +#[test] +fn test_very_long_region_names() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create regions with very long names + let long_name = "a".repeat(1000); + let region = db.create_region_if_needed(&long_name)?; + region.write(b"data")?; + + // Verify it persists + db.flush()?; + + let regions = db.regions(); + let retrieved = regions.get_from_id(&long_name); + assert!(retrieved.is_some()); + + Ok(()) +} + +#[test] +fn test_zero_byte_writes() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("empty_writes")?; + + // Write zero bytes + region.write(b"")?; + + let meta = region.meta(); + assert_eq!(meta.len(), 0); + drop(meta); + + // Write some data, then write zero bytes again + region.write(b"Hello")?; + region.write(b"")?; + + let meta = region.meta(); + assert_eq!(meta.len(), 5); + + Ok(()) +} + +#[test] +fn test_alternating_write_and_truncate() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("oscillating")?; + + for cycle in 0..10 { + // Grow + let data = vec![cycle as u8; 1000]; + region.write(&data)?; + + let meta = region.meta(); + let expected_len = if cycle == 0 { 1000 } else { 100 + 1000 }; + assert_eq!(meta.len(), expected_len); + drop(meta); + + // Shrink + region.truncate(100)?; + + let meta = region.meta(); + assert_eq!(meta.len(), 100); + drop(meta); + } + + Ok(()) +} + +// ============================================================================ +// Persistence Edge Cases +// ============================================================================ + +#[test] +fn test_retain_regions_edge_cases() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create some regions + let _ = db.create_region_if_needed("keep1")?; + let _ = db.create_region_if_needed("keep2")?; + let _ = db.create_region_if_needed("remove1")?; + + // Retain with empty set - should remove all + let empty_set = std::collections::HashSet::new(); + db.retain_regions(empty_set)?; + + let regions = db.regions(); + assert_eq!(regions.id_to_index().len(), 0); + + Ok(()) +} + +// ============================================================================ +// Layout and Hole Management +// ============================================================================ + +#[test] +fn test_complex_fragmentation_scenario() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create pattern: region, region, region, region, region + let r0 = db.create_region_if_needed("r0")?; + let r1 = db.create_region_if_needed("r1")?; + let r2 = db.create_region_if_needed("r2")?; + let r3 = db.create_region_if_needed("r3")?; + let r4 = db.create_region_if_needed("r4")?; + + for r in [&r0, &r1, &r2, &r3, &r4] { + r.write(b"data")?; + } + + // Remove pattern: keep, remove, keep, remove, keep + // This creates 2 separate holes + r1.remove()?; + r3.remove()?; + db.flush()?; // Make holes available + + let layout = db.layout(); + assert_eq!(layout.start_to_hole().len(), 2); + drop(layout); + + // Create a new region - should fill one of the holes + let r5 = db.create_region_if_needed("r5")?; + r5.write(b"fills hole")?; + + let layout = db.layout(); + assert_eq!(layout.start_to_hole().len(), 1); // One hole filled, one remains + + Ok(()) +} + +#[test] +fn test_set_min_len_preallocate() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Preallocate large file + let large_size = PAGE_SIZE * 1000; + db.set_min_len(large_size)?; + + let file_len = db.file_len(); + assert!(file_len >= large_size); + + // Should still be able to write + let region = db.create_region_if_needed("test")?; + region.write(b"After preallocation")?; + + Ok(()) +} + +// ============================================================================ +// Data Integrity Tests +// ============================================================================ + +#[test] +fn test_partial_overwrites_data_integrity() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("partial")?; + + // Write initial pattern + let initial = b"AAAAAAAAAA"; + region.write(initial)?; + + // Overwrite middle + region.write_at(b"BBB", 3)?; + + // Overwrite start + region.write_at(b"CC", 0)?; + + // Overwrite end + region.write_at(b"DD", 8)?; + + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b"CCABBBAADD"); + + Ok(()) +} + +#[test] +fn test_write_at_exact_reserved_boundary() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("boundary")?; + + // Fill exactly to PAGE_SIZE + let data = vec![42u8; PAGE_SIZE]; + region.write(&data)?; + + let meta = region.meta(); + assert_eq!(meta.len(), PAGE_SIZE); + assert_eq!(meta.reserved(), PAGE_SIZE); + drop(meta); + + // Writing one more byte should trigger expansion + region.write(b"X")?; + + let meta = region.meta(); + assert_eq!(meta.len(), PAGE_SIZE + 1); + assert!(meta.reserved() > PAGE_SIZE); + + Ok(()) +} + +// ============================================================================ +// Comprehensive Integration Test +// ============================================================================ + +#[test] +fn test_comprehensive_db_operations() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region1 = db.create_region_if_needed("region1")?; + + { + let layout = db.layout(); + + assert!(layout.start_to_region().len() == 1); + + assert!(layout.start_to_hole().is_empty()); + + let regions = db.regions(); + + assert!( + regions + .get_from_id("region1") + .is_some_and(|r| r.ptr_eq(®ion1)) + ); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 0); + assert!(region1_meta.reserved() == PAGE_SIZE); + } + + region1.write(&[0, 1, 2, 3, 4])?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 5); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!(db.mmap()[0..10] == [0, 1, 2, 3, 4, 0, 0, 0, 0, 0]); + } + + region1.write(&[5, 6, 7, 8, 9])?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 10); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!(db.mmap()[0..10] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + } + + region1.write_at(&[1, 2], 0)?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 10); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!(db.mmap()[0..10] == [1, 2, 2, 3, 4, 5, 6, 7, 8, 9]); + } + + region1.write_at(&[10, 11, 12, 13, 14, 15, 16, 17, 18], 4)?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 13); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!( + db.mmap()[0..20] + == [ + 1, 2, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 0, 0, 0, 0, 0 + ] + ); + } + + region1.write_at(&[0, 0, 0, 0, 0, 1], 13)?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 19); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!( + db.mmap()[0..20] + == [ + 1, 2, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 0, 0, 0, 1, 0 + ] + ); + } + + region1.write_at(&[1; 8000], 0)?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 8000); + assert!(region1_meta.reserved() == PAGE_SIZE * 2); + assert!(db.mmap()[0..8000] == [1; 8000]); + assert!(db.mmap()[8000..8001] == [0]); + } + + db.flush()?; + + region1.truncate(10)?; + db.compact()?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 10); + assert!(region1_meta.reserved() == PAGE_SIZE * 2); + + // We only punch a hole in whole pages (4096 bytes) + // Thus the last byte of the page where the is still data wasn't overwritten when truncating + // And the first byte of the punched page was set to 0 + assert!(db.mmap()[4095..=4096] == [1, 0]); + } + + db.flush()?; + + region1.truncate(10)?; + db.compact()?; + + { + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 10); + assert!(region1_meta.reserved() == PAGE_SIZE * 2); + // We only punch a hole in whole pages (4096 bytes) + // Thus the last byte of the page where the is still data wasn't overwritten when truncating + // And the first byte of the punched page was set to 0 + assert!(db.mmap()[4095..=4096] == [1, 0]); + } + + db.flush()?; + + region1.remove()?; + db.compact()?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 1); + assert!(index_to_region[0].is_none()); + assert!(regions.id_to_index().is_empty()); + + let layout = db.layout(); + assert!(layout.start_to_region().is_empty()); + assert!(layout.start_to_hole().len() == 1); + } + + let region1 = db.create_region_if_needed("region1")?; + let region2 = db.create_region_if_needed("region2")?; + let region3 = db.create_region_if_needed("region3")?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 0); + assert!(region1_meta.reserved() == PAGE_SIZE); + + let region2_meta = region2.meta(); + assert!(region2_meta.start() == PAGE_SIZE); + assert!(region2_meta.len() == 0); + assert!(region2_meta.reserved() == PAGE_SIZE); + + let region3_meta = region3.meta(); + assert!(region3_meta.start() == PAGE_SIZE * 2); + assert!(region3_meta.len() == 0); + assert!(region3_meta.reserved() == PAGE_SIZE); + + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 3); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2") == Some(&1)); + assert!(id_to_index.get("region3") == Some(&2)); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 3); + + assert!(start_to_index.get(&0).unwrap().ptr_eq(®ion1)); + assert!(start_to_index.get(&PAGE_SIZE).unwrap().ptr_eq(®ion2)); + assert!( + start_to_index + .get(&(PAGE_SIZE * 2)) + .unwrap() + .ptr_eq(®ion3) + ); + assert!(layout.start_to_hole().is_empty()); + } + + region2.remove()?; + db.compact()?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 0); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!(index_to_region.get(1).is_some_and(|opt| opt.is_none())); + + let region3_meta = region3.meta(); + assert!(region3_meta.start() == PAGE_SIZE * 2); + assert!(region3_meta.len() == 0); + assert!(region3_meta.reserved() == PAGE_SIZE); + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 2); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2").is_none()); + assert!(id_to_index.get("region3") == Some(&2)); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 2); + assert!(start_to_index.get(&0).unwrap().ptr_eq(®ion1)); + assert!( + start_to_index + .get(&(PAGE_SIZE * 2)) + .unwrap() + .ptr_eq(®ion3) + ); + let start_to_hole = layout.start_to_hole(); + assert!(start_to_hole.len() == 1); + assert!(start_to_hole.get(&PAGE_SIZE) == Some(&PAGE_SIZE)); + } + + let region2 = db.create_region_if_needed("region2")?; + let region2_i = region2.index(); + assert!(region2_i == 1); + + region2.remove()?; + db.compact()?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 0); + assert!(region1_meta.reserved() == PAGE_SIZE); + assert!( + index_to_region + .get(region2_i) + .is_some_and(|opt| opt.is_none()) + ); + + let region3_meta = region3.meta(); + assert!(region3_meta.start() == PAGE_SIZE * 2); + assert!(region3_meta.len() == 0); + assert!(region3_meta.reserved() == PAGE_SIZE); + + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 2); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2").is_none()); + assert!(id_to_index.get("region3") == Some(&2)); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 2); + assert!(start_to_index.get(&0).unwrap().ptr_eq(®ion1)); + assert!( + start_to_index + .get(&(PAGE_SIZE * 2)) + .unwrap() + .ptr_eq(®ion3) + ); + + let start_to_hole = layout.start_to_hole(); + assert!(start_to_hole.len() == 1); + assert!(start_to_hole.get(&PAGE_SIZE) == Some(&PAGE_SIZE)); + } + + region1.write_at(&[1; 8000], 0)?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 8000); + assert!(region1_meta.reserved() == 2 * PAGE_SIZE); + assert!( + index_to_region + .get(region2_i) + .is_some_and(|opt| opt.is_none()) + ); + + let region3_meta = region3.meta(); + assert!(region3_meta.start() == PAGE_SIZE * 2); + assert!(region3_meta.len() == 0); + assert!(region3_meta.reserved() == PAGE_SIZE); + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 2); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2").is_none()); + assert!(id_to_index.get("region3") == Some(&2)); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 2); + assert!(start_to_index.get(&0).unwrap().ptr_eq(®ion1)); + assert!( + start_to_index + .get(&(PAGE_SIZE * 2)) + .unwrap() + .ptr_eq(®ion3) + ); + let start_to_hole = layout.start_to_hole(); + assert!(start_to_hole.is_empty()); + } + + let region2 = db.create_region_if_needed("region2")?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 8000); + assert!(region1_meta.reserved() == 2 * PAGE_SIZE); + + let region2_meta = region2.meta(); + assert!(region2_meta.start() == PAGE_SIZE * 3); + assert!(region2_meta.len() == 0); + assert!(region2_meta.reserved() == PAGE_SIZE); + + let region3_meta = region3.meta(); + assert!(region3_meta.start() == PAGE_SIZE * 2); + assert!(region3_meta.len() == 0); + assert!(region3_meta.reserved() == PAGE_SIZE); + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 3); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2") == Some(&1)); + assert!(id_to_index.get("region3") == Some(&2)); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 3); + assert!(start_to_index.get(&0).unwrap().ptr_eq(®ion1)); + assert!( + start_to_index + .get(&(PAGE_SIZE * 2)) + .unwrap() + .ptr_eq(®ion3) + ); + assert!( + start_to_index + .get(&(PAGE_SIZE * 3)) + .unwrap() + .ptr_eq(®ion2) + ); + let start_to_hole = layout.start_to_hole(); + assert!(start_to_hole.is_empty()); + } + + region3.remove()?; + db.compact()?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == 0); + assert!(region1_meta.len() == 8000); + assert!(region1_meta.reserved() == 2 * PAGE_SIZE); + + let region2_meta = region2.meta(); + assert!(region2_meta.start() == PAGE_SIZE * 3); + assert!(region2_meta.len() == 0); + assert!(region2_meta.reserved() == PAGE_SIZE); + + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 2); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2") == Some(&1)); + assert!(id_to_index.get("region3").is_none()); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 2); + assert!(start_to_index.get(&0).unwrap().ptr_eq(®ion1)); + assert!( + start_to_index + .get(&(PAGE_SIZE * 3)) + .unwrap() + .ptr_eq(®ion2) + ); + let start_to_hole = layout.start_to_hole(); + assert!(start_to_hole.get(&(PAGE_SIZE * 2)) == Some(&PAGE_SIZE)); + } + + region1.write(&[1; 8000])?; + db.compact()?; + + { + let regions = db.regions(); + let index_to_region = regions.index_to_region(); + assert!(index_to_region.len() == 3); + + let region1_meta = region1.meta(); + assert!(region1_meta.start() == PAGE_SIZE * 4); + assert!(region1_meta.len() == 16_000); + assert!(region1_meta.reserved() == 4 * PAGE_SIZE); + + let region2_meta = region2.meta(); + assert!(region2_meta.start() == PAGE_SIZE * 3); + assert!(region2_meta.len() == 0); + assert!(region2_meta.reserved() == PAGE_SIZE); + + let id_to_index = regions.id_to_index(); + assert!(id_to_index.len() == 2); + assert!(id_to_index.get("region1") == Some(&0)); + assert!(id_to_index.get("region2") == Some(&1)); + assert!(id_to_index.get("region3").is_none()); + + let layout = db.layout(); + let start_to_index = layout.start_to_region(); + assert!(start_to_index.len() == 2); + assert!( + start_to_index + .get(&(PAGE_SIZE * 4)) + .unwrap() + .ptr_eq(®ion1) + ); + assert!( + start_to_index + .get(&(PAGE_SIZE * 3)) + .unwrap() + .ptr_eq(®ion2) + ); + let start_to_hole = layout.start_to_hole(); + assert!(start_to_hole.get(&0) == Some(&(PAGE_SIZE * 3))); + } + + region2.write(&[1; 6000])?; + + let region4 = db.create_region_if_needed("region4")?; + region2.remove()?; + region4.remove()?; + + Ok(()) +} + +// ============================================================================ +// Region Rename Tests +// ============================================================================ + +#[test] +fn test_basic_region_rename() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("old_name")?; + region.write(b"Test data")?; + + // Verify old name exists + { + let regions = db.regions(); + assert!(regions.get_from_id("old_name").is_some()); + assert!(regions.get_from_id("new_name").is_none()); + } + + // Rename the region + region.rename("new_name")?; + + // Verify new name exists and old name doesn't + { + let regions = db.regions(); + assert!(regions.get_from_id("old_name").is_none()); + assert!(regions.get_from_id("new_name").is_some()); + } + + // Verify data is still intact + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b"Test data"); + drop(reader); + + // Verify metadata was updated + let meta = region.meta(); + assert_eq!(meta.id(), "new_name"); + + Ok(()) +} + +#[test] +fn test_rename_with_persistence() -> Result<()> { + let temp = TempDir::new()?; + let path = temp.path(); + + // Create and rename region + { + let db = Database::open(path)?; + let region = db.create_region_if_needed("original")?; + region.write(b"Persistent data")?; + region.rename("renamed")?; + db.flush()?; + } + + // Reopen and verify rename persisted + { + let db = Database::open(path)?; + let regions = db.regions(); + + assert!(regions.get_from_id("original").is_none()); + let renamed = regions.get_from_id("renamed"); + assert!(renamed.is_some()); + + let reader = renamed.unwrap().create_reader(); + assert_eq!(reader.read_all(), b"Persistent data"); + } + + Ok(()) +} + +#[test] +fn test_rename_to_existing_name_fails() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region1 = db.create_region_if_needed("region1")?; + let _region2 = db.create_region_if_needed("region2")?; + + // Trying to rename region1 to region2 should fail + let result = region1.rename("region2"); + assert!(result.is_err()); + + // Verify region1 still has its original name + let regions = db.regions(); + assert!(regions.get_from_id("region1").is_some()); + assert!(regions.get_from_id("region2").is_some()); + + Ok(()) +} + +#[test] +fn test_rename_after_remove_and_recreate() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + // Create a region, write some data, then remove it + let region1 = db.create_region_if_needed("temp")?; + region1.write(b"Old data")?; + region1.remove()?; + + // Create a new region with the same name + let region2 = db.create_region_if_needed("temp")?; + region2.write(b"New data")?; + + // Rename the new region + region2.rename("renamed")?; + + // Verify the rename worked + let regions = db.regions(); + assert!(regions.get_from_id("temp").is_none()); + assert!(regions.get_from_id("renamed").is_some()); + + // Verify it has the new data (not old) + let reader = region2.create_reader(); + assert_eq!(reader.read_all(), b"New data"); + + Ok(()) +} + +#[test] +fn test_multiple_renames() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("name1")?; + region.write(b"Data")?; + + // Rename multiple times + region.rename("name2")?; + region.rename("name3")?; + region.rename("name4")?; + + // Verify final name + let regions = db.regions(); + assert!(regions.get_from_id("name1").is_none()); + assert!(regions.get_from_id("name2").is_none()); + assert!(regions.get_from_id("name3").is_none()); + assert!(regions.get_from_id("name4").is_some()); + + // Verify data is still intact + let reader = region.create_reader(); + assert_eq!(reader.read_all(), b"Data"); + + Ok(()) +} + +#[test] +fn test_rename_preserves_region_metadata() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("original")?; + + // Write large data to trigger expansion + let large_data = vec![42u8; PAGE_SIZE * 2]; + region.write(&large_data)?; + + // Capture metadata before rename + let (start_before, len_before, reserved_before, index_before) = { + let meta = region.meta(); + (meta.start(), meta.len(), meta.reserved(), region.index()) + }; + + // Rename + region.rename("renamed")?; + + // Verify metadata preserved (except id) + { + let meta = region.meta(); + assert_eq!(meta.id(), "renamed"); + assert_eq!(meta.start(), start_before); + assert_eq!(meta.len(), len_before); + assert_eq!(meta.reserved(), reserved_before); + assert_eq!(region.index(), index_before); + } + + // Verify data is still intact + let reader = region.create_reader(); + assert_eq!(reader.read_all(), &large_data[..]); + + Ok(()) +} + +#[test] +fn test_rename_with_special_characters() -> Result<()> { + let (db, _temp) = setup_test_db()?; + + let region = db.create_region_if_needed("simple")?; + + // Rename with special characters (but not control characters) + region.rename("name-with-dashes")?; + assert!(db.regions().get_from_id("name-with-dashes").is_some()); + + region.rename("name_with_underscores")?; + assert!(db.regions().get_from_id("name_with_underscores").is_some()); + + region.rename("name.with.dots")?; + assert!(db.regions().get_from_id("name.with.dots").is_some()); + + region.rename("name:with:colons")?; + assert!(db.regions().get_from_id("name:with:colons").is_some()); + + Ok(()) +} + +#[test] +fn test_concurrent_renames() -> Result<()> { + let temp = TempDir::new()?; + let db = Arc::new(Database::open(temp.path())?); + + // Create regions upfront + let regions: Vec<_> = (0..10) + .map(|i| db.create_region_if_needed(&format!("region_{}", i))) + .collect::>>()?; + + // Rename different regions concurrently + let handles: Vec<_> = regions + .into_iter() + .enumerate() + .map(|(i, region)| thread::spawn(move || region.rename(&format!("renamed_{}", i)))) + .collect(); + + // Wait for all renames + for handle in handles { + handle.join().unwrap()?; + } + + // Verify all renames succeeded + let regions_lock = db.regions(); + for i in 0..10 { + assert!(regions_lock.get_from_id(&format!("region_{}", i)).is_none()); + assert!( + regions_lock + .get_from_id(&format!("renamed_{}", i)) + .is_some() + ); + } + + Ok(()) +} diff --git a/crates/vecdb/.gitignore b/crates/vecdb/.gitignore new file mode 100644 index 000000000..39096ee76 --- /dev/null +++ b/crates/vecdb/.gitignore @@ -0,0 +1,5 @@ +/vecs +/raw +/compressed +/rollback* +/data diff --git a/crates/vecdb/Cargo.toml b/crates/vecdb/Cargo.toml new file mode 100644 index 000000000..070be02d3 --- /dev/null +++ b/crates/vecdb/Cargo.toml @@ -0,0 +1,69 @@ +[package] +name = "vecdb" +description = "High-performance mutable persistent vectors built on rawdb" +keywords = ["vec", "disk", "database"] +categories = ["database"] +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +readme.workspace = true + +[features] +derive = ["vecdb_derive"] +lz4 = ["dep:lz4_flex"] +pco = ["dep:pco"] +schemars = ["dep:schemars"] +serde = ["dep:serde"] +serde_json = ["serde", "dep:serde_json"] +sonic-rs = ["serde", "dep:sonic-rs"] +zerocopy = ["dep:zerocopy"] +zstd = ["dep:zstd"] + +[dependencies] +itoa = "1.0.18" +libc = { workspace = true } +log = { workspace = true } +parking_lot = { workspace = true } +rawdb = { workspace = true } +ryu = "1.0.23" +serde_json = { version = "1.0.150", features = ["float_roundtrip"], optional = true } +thiserror = "2" + +# [optional-dependencies] +lz4_flex = { workspace = true, optional = true } +pco = { workspace = true, optional = true } +schemars = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"], optional = true } +sonic-rs = { version = "0.5.8", optional = true } +vecdb_derive = { workspace = true, optional = true } +zerocopy = { version = "0.8.54", features = ["zerocopy-derive"], optional = true } +zstd = { version = "0.13.3", optional = true } + +[dev-dependencies] +sha2 = "0.11" +tempfile = { workspace = true } + +[[example]] +name = "zerocopy" +required-features = ["zerocopy"] + +[[example]] +name = "pcodec" +required-features = ["pco"] + +[[example]] +name = "pco_bench" +required-features = ["pco"] + +[[example]] +name = "io_vs_mmap" +required-features = ["pco"] + +[[example]] +name = "bench" +required-features = ["pco", "lz4", "zstd", "zerocopy"] + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/vecdb/README.md b/crates/vecdb/README.md new file mode 100644 index 000000000..25eb8ad25 --- /dev/null +++ b/crates/vecdb/README.md @@ -0,0 +1,321 @@ +# vecdb + +High-performance mutable persistent vectors built on [`rawdb`](../rawdb/README.md). + +## Features + +- **Vec-like API**: `push`, `update`, `truncate`, delete by index with sparse holes +- **Multiple storage formats**: + - **Raw**: `BytesVec`, `ZeroCopyVec` (uncompressed) + - **Compressed**: `PcoVec`, `LZ4Vec`, `ZstdVec` +- **Computed vectors**: `EagerVec` (stored computations), `LazyVecFrom1/2/3` (on-the-fly computation) +- **Rollback support**: Time-travel via stamped change deltas without full snapshots +- **Sparse deletions**: Delete elements leaving holes, no reindexing required +- **Thread-safe**: Concurrent reads with exclusive writes +- **Blazing fast**: See [benchmarks](../vecdb_bench/README.md) +- **Lazy persistence**: Changes buffered in memory, persisted only on explicit `flush()` + +## Not Suited For + +- **Key-value storage** - Use [`fjall`](https://crates.io/crates/fjall) or [`redb`](https://crates.io/crates/redb) +- **Variable-sized types** - Types like `String`, `Vec`, or dynamic structures +- **ACID transactions** - No transactional guarantees (use explicit rollback instead) + +## Install + +```bash +cargo add vecdb +``` + +## Quick Start + +```rust,no_run +use vecdb::{ + AnyStoredVec, AnyVec, BytesVec, Database, WritableVec, + ImportableVec, ReadableVec, Result, Version +}; +use std::path::Path; + +fn main() -> Result<()> { + // Open database + let db = Database::open(Path::new("data"))?; + + // Create vector with index type usize and value type u64 + let mut vec: BytesVec = + BytesVec::import(&db, "my_vec", Version::TWO)?; + + // Push values (buffered in memory) + for i in 0..1_000_000 { + vec.push(i); + } + + // Flush writes to rawdb region and syncs to disk + vec.flush()?; // Calls write() internally then flushes region + db.flush()?; // Syncs database metadata + + // Sequential scan via fold + let sum = vec.fold_range(0, vec.len(), 0u64, |acc, v| acc.wrapping_add(v)); + + // Random access via reader + let reader = vec.reader(); + for i in [500, 1000, 10] { + println!("vec[{}] = {}", i, reader.get(i)); + } + + Ok(()) +} +``` + +## Type Constraints + +vecdb works with **fixed-size types**: +- Numeric primitives: `u8`, `i32`, `f64`, etc. +- Fixed arrays: `[T; N]` +- Structs with `#[repr(C)]` +- Types implementing `zerocopy::FromBytes + zerocopy::AsBytes` (for `ZeroCopyVec`) +- Types implementing `Bytes` trait (for `BytesVec`, `LZ4Vec`, `ZstdVec`) +- Numeric types implementing `Pco` trait (for `PcoVec`) + +Use `#[derive(Bytes)]` or `#[derive(Pco)]` from `vecdb_derive` to enable custom wrapper types. + +## Vector Variants + +### Raw (Uncompressed) + +**`BytesVec`** - Custom serialization via `Bytes` trait +```rust,ignore +use vecdb::{BytesVec, Bytes}; + +#[derive(Bytes)] +struct UserId(u64); + +let mut vec: BytesVec = + BytesVec::import(&db, "users", Version::TWO)?; +``` + +**`ZeroCopyVec`** - Zero-copy mmap access (fastest random reads) +```rust,ignore +use vecdb::ZeroCopyVec; + +let mut vec: ZeroCopyVec = + ZeroCopyVec::import(&db, "raw", Version::TWO)?; +``` + +### Compressed + +**`PcoVec`** - Pcodec compression (best for numeric data, excellent compression ratios) +```rust,ignore +use vecdb::PcoVec; + +let mut vec: PcoVec = + PcoVec::import(&db, "prices", Version::TWO)?; +``` + +**`LZ4Vec`** - LZ4 compression (fast, general-purpose) +```rust,ignore +use vecdb::LZ4Vec; + +let mut vec: LZ4Vec = + LZ4Vec::import(&db, "hashes", Version::TWO)?; +``` + +**`ZstdVec`** - Zstd compression (high compression ratio, general-purpose) +```rust,ignore +use vecdb::ZstdVec; + +let mut vec: ZstdVec = + ZstdVec::import(&db, "data", Version::TWO)?; +``` + +### Computed Vectors + +**`EagerVec`** - Wraps any stored vector to enable eager computation methods + +Stores computed results on disk, incrementally updating when source data changes. Use for derived metrics, aggregations, transformations, moving averages, etc. + +```rust,ignore +use vecdb::EagerVec; + +let mut derived: EagerVec> = + EagerVec::import(&db, "derived", Version::TWO)?; + +// Compute methods store results on disk +// derived.compute_add(&source1, &source2)?; +// derived.compute_sma(&source, 20)?; +``` + +**`LazyVecFrom1/2/3<...>`** - Lazily computed vectors from 1-3 source vectors + +Values computed on-the-fly during iteration, nothing stored on disk. Use for temporary views or simple transformations. + +```rust,ignore +use vecdb::LazyVecFrom1; + +let lazy = LazyVecFrom1::init( + "computed", + Version::TWO, + Box::new(source.clone()), // ScannableBoxedVec + |_i, v| v * 2, +); + +// Computed on-the-fly via ReadableVec trait, not stored +lazy.for_each(|value| { + // ... +}); +``` + +## Core Operations + +### Write and Persistence + +```rust,ignore +// Push values (buffered in memory) +vec.push(42); +vec.push(100); + +// write() moves pushed values to storage (visible for reads) +vec.write()?; + +// flush() calls write() + region().flush() for durability +vec.flush()?; +db.flush()?; // Also flush database metadata +``` + +### Updates and Deletions + +```rust,ignore +// Update element at index (works on stored data) +vec.update(5, 999)?; + +// Delete element (creates a hole at that index) +let reader = vec.reader(); +let _ = vec.take(10, &reader); +drop(reader); + +// Holes are tracked and can be checked +if vec.holes().contains(&10) { + println!("Index 10 is a hole"); +} + +// Reading a hole returns None +let reader = vec.reader(); +assert_eq!(vec.get_with_reader(10, &reader), None); +``` + +### Rollback with Stamps + +Rollback uses stamped change deltas - lightweight compared to full snapshots. + +```rust,ignore +use vecdb::Stamp; + +// Create initial state +vec.push(100); +vec.push(200); +vec.stamped_write_with_changes(Stamp::new(1))?; + +// Make more changes +vec.push(300); +vec.update(0, 999)?; +vec.stamped_write_with_changes(Stamp::new(2))?; + +// Rollback to previous stamp (undoes changes from stamp 2) +vec.rollback()?; +assert_eq!(vec.stamp(), Stamp::new(1)); + +// Rollback before a stamp (undoes everything including stamp 1) +vec.rollback_before(Stamp::new(1))?; +assert_eq!(vec.stamp(), Stamp::new(0)); +``` + +Configure number of stamps to keep: +```rust,ignore +let options = (&db, "vec", Version::TWO) + .into() + .with_saved_stamped_changes(10); // Keep last 10 stamps +let vec = BytesVec::import_with(options)?; +``` + +## When To Use + +**Perfect for:** +- Storing large `Vec`s persistently on disk +- Append-only or append-mostly workloads +- High-speed sequential reads +- High-speed random reads (improved with `ZeroCopyVec`) +- Space-efficient storage for numeric time series (improved with `PcoVec`) +- Sparse deletions without reindexing +- Lightweight rollback without full snapshots +- Derived computations stored on disk (with `EagerVec`) + +**Not ideal for:** +- Heavy random write workloads +- Frequent insertions in the middle +- Variable-length data (strings, nested vectors) +- ACID transaction requirements +- Key-value lookups (use a proper key-value store) + +## Feature Flags + +No features are enabled by default. Enable only what you need: + +```bash +cargo add vecdb # BytesVec only, no compression or optional features +``` + +Available features: +- `pco` - Pcodec compression support (`PcoVec`) +- `zerocopy` - Zero-copy mmap access (`ZeroCopyVec`) +- `lz4` - LZ4 compression support (`LZ4Vec`) +- `zstd` - Zstd compression support (`ZstdVec`) +- `derive` - Derive macros for `Bytes` and `Pco` traits +- `serde` - Serde serialization support +- `serde_json` - JSON output using serde_json +- `sonic-rs` - Faster JSON using sonic-rs + +With Pcodec compression: +```bash +cargo add vecdb --features pco,derive +``` + +With all compression formats: +```bash +cargo add vecdb --features pco,zerocopy,lz4,zstd,derive +``` + +## Examples + +Comprehensive examples in [`examples/`](examples/): +- [`zerocopy.rs`](examples/zerocopy.rs) - ZeroCopyVec with holes, updates, and rollback +- [`pcodec.rs`](examples/pcodec.rs) - PcoVec with compression + +Run examples: +```bash +cargo run --example zerocopy --features zerocopy +cargo run --example pcodec --features pco +``` + +## Benchmarks + +> 10B sequential `u64` values (80 GB), Apple Silicon, `--release`. Compression ratios reflect sequential data — real-world ratios will vary. + +| Type | Disk | Write | Read | +|------|------|-------|------| +| `BytesVec` | 80.0 GB | 1.8 GB/s | 6.7 GB/s | +| `ZeroCopyVec` | 80.0 GB | 1.7 GB/s | 6.7 GB/s | +| `PcoVec` | 181 MB | 0.4 GB/s | 7.7 GB/s | +| `LZ4Vec` | 40.1 GB | 0.4 GB/s | 3.0 GB/s | +| `ZstdVec` | 10.4 GB | 0.5 GB/s | 1.0 GB/s | + +```bash +cargo run --release --example bench -p vecdb --features pco,lz4,zstd,zerocopy +BENCH_COUNT=100_000_000 cargo run ... # smaller dataset +``` + +### PcoVec SIMD (x86_64) + +For best `PcoVec` decompression on x86_64, enable BMI and AVX2: + +```bash +RUSTFLAGS="-C target-feature=+bmi1,+bmi2,+avx2" cargo build --release +``` diff --git a/crates/vecdb/examples/bench.rs b/crates/vecdb/examples/bench.rs new file mode 100644 index 000000000..8cce6afff --- /dev/null +++ b/crates/vecdb/examples/bench.rs @@ -0,0 +1,639 @@ +use std::time::{Duration, Instant}; + +use vecdb::{ + AnyStoredVec, BytesVec, Database, ImportableVec, LZ4Vec, PcoVec, ReadableVec, StoredVec, + Version, WritableVec, ZeroCopyVec, ZstdVec, +}; + +const DEFAULT_VALUE_COUNT: usize = 10_000_000_000; // 10B u64s = 80 GB +const BATCH_SIZE: usize = 10_000_000; +const MAX_RANGE_BYTES: usize = 8 * 1024 * 1024 * 1024; // 8 GB + +fn value_count() -> usize { + std::env::var("BENCH_COUNT") + .ok() + .and_then(|s| s.replace('_', "").parse().ok()) + .unwrap_or(DEFAULT_VALUE_COUNT) +} + +fn range_passes() -> usize { + std::env::var("BENCH_PASSES") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(5) +} + +fn range_sizes(count: usize, value_size: usize) -> Vec { + let max_elements = (MAX_RANGE_BYTES / value_size).min(count); + let mut sizes = Vec::new(); + let mut n = 1_000; + while n <= max_elements { + sizes.push(n); + let next5 = n * 5; + if next5 <= max_elements && next5 != n * 10 { + sizes.push(next5); + } + n *= 10; + } + sizes +} + +fn repetitions(range_size: usize) -> usize { + match range_size { + n if n < 10_000 => 5_000, + n if n < 100_000 => 1_000, + n if n < 1_000_000 => 100, + n if n < 10_000_000 => 20, + n if n < 100_000_000 => 5, + _ => 1, + } +} + +fn xorshift(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state +} + +fn random_starts(count: usize, max_start: usize) -> Vec { + let mut rng = 42u64; + (0..count) + .map(|_| xorshift(&mut rng) as usize % max_start.max(1)) + .collect() +} + +fn format_bytes(bytes: usize) -> String { + if bytes >= 1_000_000_000 { + format!("{:.1} GB", bytes as f64 / 1e9) + } else if bytes >= 1_000_000 { + format!("{:.1} MB", bytes as f64 / 1e6) + } else if bytes >= 1_000 { + format!("{:.1} KB", bytes as f64 / 1e3) + } else { + format!("{} B", bytes) + } +} + +fn format_duration(d: Duration) -> String { + let ns = d.as_nanos(); + if ns >= 1_000_000_000 { + format!("{:.2} s", d.as_secs_f64()) + } else if ns >= 1_000_000 { + format!("{:.2} ms", ns as f64 / 1e6) + } else if ns >= 1_000 { + format!("{:.2} us", ns as f64 / 1e3) + } else { + format!("{} ns", ns) + } +} + +fn throughput_str(bytes: usize, d: Duration) -> String { + format!("{:.1} GB/s", bytes as f64 / d.as_secs_f64() / 1e9) +} + +// --- Page cache eviction --- + +fn drop_caches() -> bool { + #[cfg(target_os = "macos")] + { + std::process::Command::new("purge") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + } + #[cfg(target_os = "linux")] + { + std::process::Command::new("sync").status().ok(); + std::fs::write("/proc/sys/vm/drop_caches", "3").is_ok() + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + false + } +} + +fn check_cache_eviction() -> bool { + let ok = drop_caches(); + if ok { + println!(" Cache eviction: available (cold-cache benchmarks)"); + } else { + println!(" Cache eviction: unavailable (warm-cache benchmarks)"); + println!(" Hint: run with sudo for cold-cache results"); + } + ok +} + +// --- Vec size --- + +fn print_vec_size(vec: &dyn AnyStoredVec, label: &str) { + let region_bytes = vec.region().meta().len(); + let logical_bytes = vec.len() * vec.value_type_to_size_of(); + let ratio = if logical_bytes > 0 { + region_bytes as f64 / logical_bytes as f64 + } else { + 0.0 + }; + eprintln!( + " {label} on disk: {} (logical: {}, ratio: {:.2}x)", + format_bytes(region_bytes), + format_bytes(logical_bytes), + ratio, + ); +} + +// --- Populate --- + +fn populate + ImportableVec + AnyStoredVec>( + db: &Database, + label: &str, + count: usize, +) -> V { + eprint!(" Populating {label} with {count} values..."); + flush(); + let start = Instant::now(); + let mut vec: V = V::import(db, "bench", Version::ONE).unwrap(); + let mut written = 0; + while written < count { + let end = (written + BATCH_SIZE).min(count); + for i in written..end { + vec.push(i as u64); + } + vec.write().unwrap(); + written = end; + eprint!( + "\r Populating {label}: {:.0}% ", + written as f64 / count as f64 * 100.0 + ); + } + db.flush().unwrap(); + eprintln!("\r Populated {label} ({:?}) ", start.elapsed()); + print_vec_size(&vec, label); + vec +} + +// --- Benchmark helpers --- + +// --- Benchmark runners --- + +struct BenchResult { + name: &'static str, + duration: Duration, +} + +fn print_full_results(results: &[BenchResult], total_bytes: usize) { + let best = results.iter().min_by_key(|r| r.duration).unwrap().duration; + for r in results { + let pct = if r.duration > best { + let overhead = (r.duration.as_secs_f64() / best.as_secs_f64() - 1.0) * 100.0; + format!("(+{overhead:.0}%)") + } else { + String::new() + }; + println!( + " {:<20} {} ({}) {}", + r.name, + format_duration(r.duration), + throughput_str(total_bytes, r.duration), + pct, + ); + } +} + +fn print_range_header(columns: &[&str]) { + print!("{:>12} {:>10}", "range", "bytes"); + for col in columns { + print!(" {:>14}", col); + } + println!(" {:<8}", "winner"); + let width = 24 + columns.len() * 15 + 10; + println!("{}", "-".repeat(width)); +} + +fn print_range_row(range_size: usize, results: &[BenchResult]) { + let range_bytes = range_size * 8; + print!("{:>12} {:>10}", range_size, format_bytes(range_bytes)); + let best = results.iter().min_by_key(|r| r.duration).unwrap(); + for r in results { + print!(" {:>14}", format_duration(r.duration)); + } + println!(" {:<8}", best.name); +} + +// --- StoredFold trait for IO/mmap access --- + +trait StoredFold { + fn fold_stored_io_sum(&self, from: usize, to: usize, acc: u64) -> u64; + fn fold_stored_mmap_sum(&self, from: usize, to: usize, acc: u64) -> u64; +} + +impl StoredFold for BytesVec { + fn fold_stored_io_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_io(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } + fn fold_stored_mmap_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_mmap(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } +} + +impl StoredFold for ZeroCopyVec { + fn fold_stored_io_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_io(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } + fn fold_stored_mmap_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_mmap(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } +} + +impl StoredFold for LZ4Vec { + fn fold_stored_io_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_io(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } + fn fold_stored_mmap_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_mmap(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } +} + +impl StoredFold for PcoVec { + fn fold_stored_io_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_io(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } + fn fold_stored_mmap_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_mmap(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } +} + +impl StoredFold for ZstdVec { + fn fold_stored_io_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_io(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } + fn fold_stored_mmap_sum(&self, from: usize, to: usize, acc: u64) -> u64 { + self.fold_stored_mmap(from, to, acc, |a, v: u64| a.wrapping_add(v)) + } +} + +// --- Unified benchmark --- + +fn bench_vec< + V: ReadableVec + StoredFold + AnyStoredVec + StoredVec, +>( + vec: &V, + label: &str, + count: usize, + can_purge: bool, +) { + let total_bytes = count * 8; + let disk_bytes = vec.region().meta().len(); + let ratio = if total_bytes > 0 { + disk_bytes as f64 / total_bytes as f64 + } else { + 0.0 + }; + let ranges = range_sizes(count, 8); + + println!( + "\n=== {label} — {count} values ({}, disk: {}, {:.2}x) ===\n", + format_bytes(total_bytes), + format_bytes(disk_bytes), + ratio, + ); + + // Full scan — multiple passes in random method order to eliminate cache bias. + let full_passes = range_passes(); + println!("--- Full scan ({full_passes} passes, random order) ---"); + + let method_names: [&str; 6] = [ + "fold_stored_io", + "fold_stored_mmap", + "fold_range", + "try_fold_range", + "for_each_dyn", + "for_each", + ]; + let mut times = [Duration::ZERO; 6]; + let mut sum = 0u64; + let mut rng = 0xCAFEu64; + + for _pass in 0..full_passes { + let mut order = [0usize, 1, 2, 3, 4, 5]; + for i in (1..6).rev() { + let j = xorshift(&mut rng) as usize % (i + 1); + order.swap(i, j); + } + + for &method in &order { + if can_purge { + drop_caches(); + } + let t = Instant::now(); + match method { + 0 => sum = vec.fold_stored_io_sum(0, count, sum), + 1 => sum = vec.fold_stored_mmap_sum(0, count, sum), + 2 => sum = vec.fold_range(0, count, sum, |a, v: u64| a.wrapping_add(v)), + 3 => { + sum = vec + .try_fold_range(0, count, sum, |a, v: u64| Ok::<_, ()>(a.wrapping_add(v))) + .unwrap() + } + 4 => vec.for_each_range_dyn(0, count, &mut |v: u64| sum = sum.wrapping_add(v)), + 5 => vec.for_each_range(0, count, |v: u64| sum = sum.wrapping_add(v)), + _ => unreachable!(), + } + times[method] += t.elapsed(); + } + } + + std::hint::black_box(sum); + + let results: Vec = (0..6) + .map(|i| BenchResult { + name: method_names[i], + duration: times[i] / full_passes as u32, + }) + .collect(); + print_full_results(&results, total_bytes); + + // Range scans — multiple passes in random method order to eliminate cache bias. + let passes = range_passes(); + println!("\n--- Range scans ({passes} passes, random order) ---"); + let columns: Vec<&str> = vec!["IO", "Mmap", "fold", "try_fold", "dyn", "static"]; + print_range_header(&columns); + + for &range_size in &ranges { + let reps = repetitions(range_size); + let max_start = count.saturating_sub(range_size); + let starts = random_starts(reps, max_start); + + let mut times = [Duration::ZERO; 6]; + let mut sum = 0u64; + let mut rng = range_size as u64 ^ 0xDEAD; + + for _pass in 0..passes { + // Shuffle method order using Fisher-Yates. + let mut order = [0usize, 1, 2, 3, 4, 5]; + for i in (1..6).rev() { + let j = xorshift(&mut rng) as usize % (i + 1); + order.swap(i, j); + } + + for &method in &order { + if can_purge { + drop_caches(); + } + let t = Instant::now(); + for &s in &starts { + let from = s; + let to = s + range_size; + match method { + 0 => sum = vec.fold_stored_io_sum(from, to, sum), + 1 => sum = vec.fold_stored_mmap_sum(from, to, sum), + 2 => sum = vec.fold_range(from, to, sum, |a, v: u64| a.wrapping_add(v)), + 3 => { + sum = vec + .try_fold_range(from, to, sum, |a, v: u64| { + Ok::<_, ()>(a.wrapping_add(v)) + }) + .unwrap() + } + 4 => vec + .for_each_range_dyn(from, to, &mut |v: u64| sum = sum.wrapping_add(v)), + 5 => vec.for_each_range(from, to, |v: u64| sum = sum.wrapping_add(v)), + _ => unreachable!(), + } + } + times[method] += t.elapsed(); + } + } + + std::hint::black_box(sum); + + let divisor = (reps * passes) as u32; + let row = vec![ + BenchResult { + name: "IO", + duration: times[0] / divisor, + }, + BenchResult { + name: "Mmap", + duration: times[1] / divisor, + }, + BenchResult { + name: "fold", + duration: times[2] / divisor, + }, + BenchResult { + name: "try_fold", + duration: times[3] / divisor, + }, + BenchResult { + name: "dyn", + duration: times[4] / divisor, + }, + BenchResult { + name: "static", + duration: times[5] / divisor, + }, + ]; + print_range_row(range_size, &row); + } + + // --- Read-only clone benchmarks --- + + let ro = vec.read_only_clone(); + + let ro_passes = range_passes(); + println!("\n--- Read-only full scan ({ro_passes} passes, random order) ---"); + + let ro_method_names: [&str; 4] = [ + "ro_fold_range", + "ro_try_fold_range", + "ro_for_each_dyn", + "ro_for_each", + ]; + let mut ro_times = [Duration::ZERO; 4]; + let mut sum = 0u64; + let mut rng = 0xBEEFu64; + + for _pass in 0..ro_passes { + let mut order = [0usize, 1, 2, 3]; + for i in (1..4).rev() { + let j = xorshift(&mut rng) as usize % (i + 1); + order.swap(i, j); + } + + for &method in &order { + if can_purge { + drop_caches(); + } + let t = Instant::now(); + match method { + 0 => sum = ro.fold_range(0, count, sum, |a, v: u64| a.wrapping_add(v)), + 1 => { + sum = ro + .try_fold_range(0, count, sum, |a, v: u64| Ok::<_, ()>(a.wrapping_add(v))) + .unwrap() + } + 2 => ro.for_each_range_dyn(0, count, &mut |v: u64| sum = sum.wrapping_add(v)), + 3 => ro.for_each_range(0, count, |v: u64| sum = sum.wrapping_add(v)), + _ => unreachable!(), + } + ro_times[method] += t.elapsed(); + } + } + + std::hint::black_box(sum); + + let ro_results: Vec = (0..4) + .map(|i| BenchResult { + name: ro_method_names[i], + duration: ro_times[i] / ro_passes as u32, + }) + .collect(); + print_full_results(&ro_results, total_bytes); + + let ro_passes = range_passes(); + println!("\n--- Read-only range scans ({ro_passes} passes, random order) ---"); + let ro_columns: Vec<&str> = vec!["fold", "try_fold", "dyn", "static"]; + print_range_header(&ro_columns); + + for &range_size in &ranges { + let reps = repetitions(range_size); + let max_start = count.saturating_sub(range_size); + let starts = random_starts(reps, max_start); + + let mut times = [Duration::ZERO; 4]; + let mut sum = 0u64; + let mut rng = range_size as u64 ^ 0xFACE; + + for _pass in 0..ro_passes { + let mut order = [0usize, 1, 2, 3]; + for i in (1..4).rev() { + let j = xorshift(&mut rng) as usize % (i + 1); + order.swap(i, j); + } + + for &method in &order { + if can_purge { + drop_caches(); + } + let t = Instant::now(); + for &s in &starts { + let from = s; + let to = s + range_size; + match method { + 0 => sum = ro.fold_range(from, to, sum, |a, v: u64| a.wrapping_add(v)), + 1 => { + sum = ro + .try_fold_range(from, to, sum, |a, v: u64| { + Ok::<_, ()>(a.wrapping_add(v)) + }) + .unwrap() + } + 2 => { + ro.for_each_range_dyn(from, to, &mut |v: u64| sum = sum.wrapping_add(v)) + } + 3 => ro.for_each_range(from, to, |v: u64| sum = sum.wrapping_add(v)), + _ => unreachable!(), + } + } + times[method] += t.elapsed(); + } + } + + std::hint::black_box(sum); + + let divisor = (reps * ro_passes) as u32; + let row = vec![ + BenchResult { + name: "fold", + duration: times[0] / divisor, + }, + BenchResult { + name: "try_fold", + duration: times[1] / divisor, + }, + BenchResult { + name: "dyn", + duration: times[2] / divisor, + }, + BenchResult { + name: "static", + duration: times[3] / divisor, + }, + ]; + print_range_row(range_size, &row); + } +} + +/// Fixed bench directory — cleaned up at start so Ctrl+C leftovers don't accumulate. +fn bench_dir() -> std::path::PathBuf { + std::env::temp_dir().join("vecdb_bench") +} + +fn cleanup_bench_dir() { + let dir = bench_dir(); + if dir.exists() { + eprint!(" Cleaning up previous bench data..."); + flush(); + std::fs::remove_dir_all(&dir).ok(); + eprintln!(" done"); + } +} + +fn run_bench< + V: ReadableVec + StoredFold + AnyStoredVec + StoredVec, +>( + label: &str, + count: usize, + can_purge: bool, +) { + cleanup_bench_dir(); + let dir = bench_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let db = Database::open(&dir).unwrap(); + let vec = populate::(&db, label, count); + bench_vec(&vec, label, count, can_purge); + drop(vec); + drop(db); + std::fs::remove_dir_all(&dir).ok(); +} + +fn main() { + let args: Vec = std::env::args().collect(); + let mode = args.get(1).map(|s| s.as_str()).unwrap_or("all"); + let count = value_count(); + + println!("BENCH_COUNT={count} (set env to override, e.g. BENCH_COUNT=1_000_000)"); + let can_purge = check_cache_eviction(); + println!(); + + let run_bytes = matches!(mode, "all" | "bytes" | "raw"); + let run_zerocopy = matches!(mode, "all" | "zerocopy" | "raw"); + let run_lz4 = matches!(mode, "all" | "lz4" | "compressed"); + let run_pco = matches!(mode, "all" | "pco" | "compressed"); + let run_zstd = matches!(mode, "all" | "zstd" | "compressed"); + + if run_bytes { + run_bench::>("BytesVec", count, can_purge); + } + if run_zerocopy { + run_bench::>("ZeroCopyVec", count, can_purge); + } + if run_lz4 { + run_bench::>("LZ4Vec", count, can_purge); + } + if run_pco { + run_bench::>("PcoVec", count, can_purge); + } + if run_zstd { + run_bench::>("ZstdVec", count, can_purge); + } + + cleanup_bench_dir(); +} + +fn flush() { + std::io::Write::flush(&mut std::io::stderr()).ok(); + std::io::Write::flush(&mut std::io::stdout()).ok(); +} diff --git a/crates/vecdb/examples/io_vs_mmap.rs b/crates/vecdb/examples/io_vs_mmap.rs new file mode 100644 index 000000000..167d2eb13 --- /dev/null +++ b/crates/vecdb/examples/io_vs_mmap.rs @@ -0,0 +1,284 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + thread, + time::{Duration, Instant}, +}; + +use vecdb::{ + AnyStoredVec, AnyVec, BytesVec, Database, ImportableVec, PcoVec, ReadableVec, Version, + WritableVec, +}; + +const VALUE_COUNT: usize = 10_000_000_000; // 10B u64s = 80GB +const BATCH_SIZE: usize = 100_000_000; +const SEED: u64 = 42; + +const RANGE_SIZES: &[usize] = &[1_000_000, 10_000_000, 50_000_000, 100_000_000, 500_000_000]; +const THREAD_COUNTS: &[usize] = &[1, 2, 4, 8]; + +fn repetitions(range_size: usize) -> usize { + match range_size { + n if n < 10_000 => 10_000, + n if n < 1_000_000 => 1_000, + n if n < 10_000_000 => 100, + _ => 10, + } +} + +fn xorshift(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state +} + +fn random_starts(count: usize, max_start: usize) -> Vec { + let mut rng = SEED; + (0..count) + .map(|_| xorshift(&mut rng) as usize % max_start.max(1)) + .collect() +} + +// --- Sequential benchmark --- + +fn bench_fold(vec: &V, range_size: usize, starts: &[usize]) -> Duration +where + V: ReadableVec, +{ + let reps = starts.len(); + let mut sum = 0u64; + let start = Instant::now(); + for &s in starts { + sum = vec.fold_range(s, s + range_size, sum, |acc, v| acc.wrapping_add(v)); + } + let elapsed = start.elapsed(); + std::hint::black_box(sum); + elapsed / reps as u32 +} + +// --- Parallel benchmark --- + +fn bench_par_fold(vec: &V, range_size: usize, starts: &[usize], threads: usize) -> Duration +where + V: ReadableVec + Sync, +{ + let reps = starts.len(); + let chunk_size = reps.div_ceil(threads); + let sum = Arc::new(AtomicU64::new(0)); + + let start = Instant::now(); + thread::scope(|s| { + for chunk in starts.chunks(chunk_size) { + let sum = Arc::clone(&sum); + s.spawn(move || { + let mut local_sum = 0u64; + for &st in chunk { + local_sum = vec + .fold_range(st, st + range_size, local_sum, |acc, v| acc.wrapping_add(v)); + } + sum.fetch_add(local_sum, Ordering::Relaxed); + }); + } + }); + let elapsed = start.elapsed(); + std::hint::black_box(sum.load(Ordering::Relaxed)); + elapsed / reps as u32 +} + +// --- Output formatting --- + +fn print_header() { + println!("{:>12} {:>10} {:>14}", "range_size", "bytes", "fold/iter"); + println!("{}", "-".repeat(40)); +} + +fn print_par_header() { + print!("{:>12} {:>10}", "range_size", "bytes"); + for &t in THREAD_COUNTS { + print!(" {:>12}", format!("{t}T")); + } + println!(); + println!("{}", "-".repeat(12 + 10 + THREAD_COUNTS.len() * 13)); +} + +fn print_row(range_size: usize, per: Duration) { + let bytes = range_size * 8; + println!( + "{:>12} {:>10} {:>14}", + range_size, + format_bytes(bytes), + format_duration(per), + ); +} + +fn print_par_row(range_size: usize, times: &[Duration]) { + let bytes = range_size * 8; + print!("{:>12} {:>10}", range_size, format_bytes(bytes)); + for d in times { + print!(" {:>12}", format_duration(*d)); + } + println!(); +} + +fn format_bytes(bytes: usize) -> String { + if bytes >= 1_000_000_000 { + format!("{:.1} GB", bytes as f64 / 1e9) + } else if bytes >= 1_000_000 { + format!("{:.1} MB", bytes as f64 / 1e6) + } else if bytes >= 1_000 { + format!("{:.1} KB", bytes as f64 / 1e3) + } else { + format!("{} B", bytes) + } +} + +fn format_duration(d: Duration) -> String { + let ns = d.as_nanos(); + if ns >= 1_000_000_000 { + format!("{:.2} s", d.as_secs_f64()) + } else if ns >= 1_000_000 { + format!("{:.2} ms", ns as f64 / 1e6) + } else if ns >= 1_000 { + format!("{:.2} us", ns as f64 / 1e3) + } else { + format!("{} ns", ns) + } +} + +// --- Populate --- + +fn populate_bytes(dir: &std::path::Path) { + eprint!("Populating BytesVec with {VALUE_COUNT} u64s (80 GB)..."); + flush(); + let pop_start = Instant::now(); + let db = Database::open(dir).unwrap(); + let mut vec: BytesVec = BytesVec::import(&db, "bench", Version::ONE).unwrap(); + let mut written = 0; + while written < VALUE_COUNT { + let end = (written + BATCH_SIZE).min(VALUE_COUNT); + for i in written..end { + vec.push(i as u64); + } + vec.write().unwrap(); + written = end; + let elapsed = pop_start.elapsed(); + let gb = written * 8 / 1_000_000_000; + let gbs = gb as f64 / elapsed.as_secs_f64(); + eprint!( + "\r {:.0}% - {gb} GB - {gbs:.1} GB/s ", + written as f64 / VALUE_COUNT as f64 * 100.0 + ); + } + db.flush().unwrap(); + eprintln!("\n done ({:?})", pop_start.elapsed()); +} + +fn populate_pco(dir: &std::path::Path) { + eprint!("Populating PcoVec with {VALUE_COUNT} u64s..."); + flush(); + let pop_start = Instant::now(); + let db = Database::open(dir).unwrap(); + let mut vec: PcoVec = PcoVec::import(&db, "bench", Version::ONE).unwrap(); + let mut written = 0; + while written < VALUE_COUNT { + let end = (written + BATCH_SIZE).min(VALUE_COUNT); + for i in written..end { + vec.push(i as u64); + } + vec.write().unwrap(); + written = end; + let elapsed = pop_start.elapsed(); + let pct = written as f64 / VALUE_COUNT as f64 * 100.0; + eprint!("\r {pct:.0}% ({:?}) ", elapsed); + } + db.flush().unwrap(); + eprintln!("\n done ({:?})", pop_start.elapsed()); +} + +// --- Bench per type --- + +fn bench_type(vec: &V, label: &str) +where + V: ReadableVec + AnyVec + Sync, +{ + println!( + "\n=== {label} — {} values ({} GB) ===\n", + VALUE_COUNT, + VALUE_COUNT * 8 / 1_000_000_000 + ); + + // Sequential + println!("--- Sequential ---"); + print_header(); + for &range_size in RANGE_SIZES { + let reps = repetitions(range_size); + let max_start = VALUE_COUNT.saturating_sub(range_size); + let starts = random_starts(reps, max_start); + let per = bench_fold(vec, range_size, &starts); + print_row(range_size, per); + } + + // Parallel + println!("\n--- Parallel ---"); + print_par_header(); + for &range_size in RANGE_SIZES { + let reps = repetitions(range_size); + let max_start = VALUE_COUNT.saturating_sub(range_size); + let starts = random_starts(reps, max_start); + + let times: Vec = THREAD_COUNTS + .iter() + .map(|&t| bench_par_fold(vec, range_size, &starts, t)) + .collect(); + print_par_row(range_size, ×); + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let mode = args.get(1).map(|s| s.as_str()).unwrap_or("both"); + + match mode { + "bytes" => { + let dir = tempfile::tempdir().unwrap(); + populate_bytes(dir.path()); + let db = Database::open(dir.path()).unwrap(); + let vec: BytesVec = BytesVec::import(&db, "bench", Version::ONE).unwrap(); + bench_type(&vec, "BytesVec"); + } + "pco" => { + let dir = tempfile::tempdir().unwrap(); + populate_pco(dir.path()); + let db = Database::open(dir.path()).unwrap(); + let vec: PcoVec = PcoVec::import(&db, "bench", Version::ONE).unwrap(); + bench_type(&vec, "PcoVec"); + } + _ => { + let bytes_dir = tempfile::tempdir().unwrap(); + populate_bytes(bytes_dir.path()); + { + let db = Database::open(bytes_dir.path()).unwrap(); + let vec: BytesVec = + BytesVec::import(&db, "bench", Version::ONE).unwrap(); + bench_type(&vec, "BytesVec"); + } + drop(bytes_dir); + + let pco_dir = tempfile::tempdir().unwrap(); + populate_pco(pco_dir.path()); + { + let db = Database::open(pco_dir.path()).unwrap(); + let vec: PcoVec = PcoVec::import(&db, "bench", Version::ONE).unwrap(); + bench_type(&vec, "PcoVec"); + } + drop(pco_dir); + } + } +} + +fn flush() { + std::io::Write::flush(&mut std::io::stdout()).ok(); +} diff --git a/crates/vecdb/examples/iter_vs_mmap.rs b/crates/vecdb/examples/iter_vs_mmap.rs new file mode 100644 index 000000000..8b7a37a31 --- /dev/null +++ b/crates/vecdb/examples/iter_vs_mmap.rs @@ -0,0 +1,89 @@ +use std::time::Instant; + +use vecdb::{AnyStoredVec, BytesVec, Database, ImportableVec, ReadableVec, Version, WritableVec}; + +const VALUE_COUNT: usize = 10_000_000_000; // 10B u64s = 80GB +const BATCH_SIZE: usize = 100_000_000; +const RANGE_SIZE: usize = 1_000_000; +const REPEATS: usize = 100; +const SEED: u64 = 42; + +fn main() { + let dir = tempfile::tempdir().unwrap(); + let db = Database::open(dir.path()).unwrap(); + let mut vec: BytesVec = BytesVec::import(&db, "bench", Version::TWO).unwrap(); + + // --- Write 80GB --- + println!( + "Writing {VALUE_COUNT} values ({} GB)...", + VALUE_COUNT * 8 / 1_000_000_000 + ); + let write_start = Instant::now(); + let mut written = 0usize; + while written < VALUE_COUNT { + let batch_end = (written + BATCH_SIZE).min(VALUE_COUNT); + for i in written..batch_end { + vec.push(i as u64); + } + vec.write().unwrap(); + written = batch_end; + let elapsed = write_start.elapsed(); + let pct = written as f64 / VALUE_COUNT as f64 * 100.0; + let gb = written * 8 / 1_000_000_000; + let gbs = gb as f64 / elapsed.as_secs_f64(); + eprint!("\r {pct:.0}% - {gb} GB - {gbs:.1} GB/s "); + } + db.flush().unwrap(); + eprintln!(); + println!("Write done in {:?}\n", write_start.elapsed()); + + // === Full sequential read (80GB) === + println!("=== Full sequential read (80 GB) ===\n"); + + { + print!(" fold_range ... "); + flush(); + let start = Instant::now(); + let sum = vec.fold_range(0, VALUE_COUNT, 0u64, |acc, v: u64| acc.wrapping_add(v)); + std::hint::black_box(sum); + let elapsed = start.elapsed(); + println!("{elapsed:?} ({:.2} GB/s)", 80.0 / elapsed.as_secs_f64()); + } + + // === Range reads (fixed + random-start) x 100 === + let fixed_from = VALUE_COUNT / 2; + let mut rng_state: u64 = SEED; + let random_starts: Vec = (0..REPEATS) + .map(|_| { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 7; + rng_state ^= rng_state << 17; + (rng_state as usize) % (VALUE_COUNT - RANGE_SIZE) + }) + .collect(); + + println!("\n=== {REPEATS}x (fixed 1M + random-start 1M) ===\n"); + + { + print!(" fold_range ... "); + flush(); + let start = Instant::now(); + let mut sum = 0u64; + (0..REPEATS).for_each(|i| { + sum = vec.fold_range(fixed_from, fixed_from + RANGE_SIZE, sum, |acc, v: u64| { + acc.wrapping_add(v) + }); + let from = random_starts[i]; + sum = vec.fold_range(from, from + RANGE_SIZE, sum, |acc, v: u64| { + acc.wrapping_add(v) + }); + }); + std::hint::black_box(sum); + let elapsed = start.elapsed(); + println!("{elapsed:?} ({:?}/iter)", elapsed / REPEATS as u32); + } +} + +fn flush() { + std::io::Write::flush(&mut std::io::stdout()).ok(); +} diff --git a/crates/vecdb/examples/pco_bench.rs b/crates/vecdb/examples/pco_bench.rs new file mode 100644 index 000000000..eb2fc2b1a --- /dev/null +++ b/crates/vecdb/examples/pco_bench.rs @@ -0,0 +1,104 @@ +use std::time::Instant; + +use pco::{ + ChunkConfig, + standalone::{simple_compress, simple_decompress, simple_decompress_into}, +}; + +fn main() { + // Generate test data - different patterns + let sequential: Vec = (0..1_000_000).collect(); + let repeated: Vec = (0..1_000_000).map(|i| (i % 1000) as u32).collect(); + let random: Vec = (0..1_000_000) + .map(|i| ((i * 17 + 31) % 100_000) as u32) + .collect(); + + let datasets = [ + ("sequential", &sequential), + ("repeated", &repeated), + ("pseudo-random", &random), + ]; + + let levels = [0, 4, 8, 12]; + + println!("Benchmarking pco compression levels (1M u32 values)\n"); + println!( + "{:<15} {:>6} {:>12} {:>12} {:>12} {:>10}", + "Dataset", "Level", "Compress", "Decompress", "Size", "Ratio" + ); + println!("{}", "-".repeat(75)); + + for (name, data) in &datasets { + let raw_size = data.len() * 4; + + for &level in &levels { + let config = ChunkConfig::default() + .with_compression_level(level) + .with_enable_8_bit(true); + + // Benchmark compression + let start = Instant::now(); + let compressed = simple_compress(data, &config).unwrap(); + let compress_time = start.elapsed(); + + // Benchmark decompression + let start = Instant::now(); + let _decompressed: Vec = simple_decompress(&compressed).unwrap(); + let decompress_time = start.elapsed(); + + let ratio = raw_size as f64 / compressed.len() as f64; + + println!( + "{:<15} {:>6} {:>10.2}ms {:>10.2}ms {:>10} {:>10.2}x", + name, + level, + compress_time.as_secs_f64() * 1000.0, + decompress_time.as_secs_f64() * 1000.0, + compressed.len(), + ratio + ); + } + println!(); + } + + // Benchmark decompress vs decompress_into (buffer reuse) + println!("\nBenchmarking decompress vs decompress_into (buffer reuse)\n"); + println!( + "{:<15} {:>15} {:>15} {:>10}", + "Dataset", "decompress", "decompress_into", "Speedup" + ); + println!("{}", "-".repeat(60)); + + let config = ChunkConfig::default().with_enable_8_bit(true); + let iterations = 100; + + for (name, data) in &datasets { + let compressed = simple_compress(data, &config).unwrap(); + let len = data.len(); + + // Benchmark simple_decompress (allocates each time) + let start = Instant::now(); + for _ in 0..iterations { + let _: Vec = simple_decompress(&compressed).unwrap(); + } + let alloc_time = start.elapsed(); + + // Benchmark simple_decompress_into (reuses buffer) + let mut buffer: Vec = vec![0; len]; + let start = Instant::now(); + for _ in 0..iterations { + simple_decompress_into(&compressed, &mut buffer).unwrap(); + } + let reuse_time = start.elapsed(); + + let speedup = alloc_time.as_secs_f64() / reuse_time.as_secs_f64(); + + println!( + "{:<15} {:>13.2}ms {:>13.2}ms {:>10.2}x", + name, + alloc_time.as_secs_f64() * 1000.0 / iterations as f64, + reuse_time.as_secs_f64() * 1000.0 / iterations as f64, + speedup + ); + } +} diff --git a/crates/vecdb/examples/pcodec.rs b/crates/vecdb/examples/pcodec.rs new file mode 100644 index 000000000..59bc896ba --- /dev/null +++ b/crates/vecdb/examples/pcodec.rs @@ -0,0 +1,172 @@ +use std::{fs, path::Path}; + +use vecdb::{ + AnyStoredVec, AnyVec, Database, ImportableVec, PcoVec, ReadableVec, Stamp, Version, WritableVec, +}; + +#[allow(clippy::upper_case_acronyms)] +type VEC = PcoVec; + +fn main() -> Result<(), Box> { + let _ = fs::remove_dir_all("compressed"); + + let version = Version::TWO; + + let database = Database::open(Path::new("compressed"))?; + + let options = (&database, "vec", version).into(); + + { + let mut vec: VEC = PcoVec::forced_import_with(options)?; + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + vec.write()?; + + assert_eq!(vec.header().stamp(), Stamp::new(0)); + } + + { + let mut vec: VEC = PcoVec::forced_import_with(options)?; + + vec.mut_header().update_stamp(Stamp::new(100)); + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(3, 4), vec![3]); + assert_eq!(vec.collect_range(4, 5), vec![4]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(0, 1), vec![0]); + + vec.push(21); + vec.push(22); + + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.pushed_len(), 2); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + assert!(vec.collect_range(23, 24).is_empty()); + + vec.write()?; + } + + { + let mut vec: VEC = PcoVec::forced_import_with(options)?; + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.stored_len(), 23); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + + vec.truncate_if_needed(14)?; + + assert_eq!(vec.stored_len(), 14); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 14); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert!(vec.collect_range(20, 21).is_empty()); + + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![9, 10, 11, 12, 13] + ); + + vec.push(vec.len() as u32); + let all = vec.collect(); + assert_eq!(*all.last().unwrap(), 14); + + vec.write()?; + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + } + + { + let mut vec: VEC = PcoVec::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert!(vec.collect_range(20, 21).is_empty()); + + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![10, 11, 12, 13, 14] + ); + + vec.reset()?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 0); + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.pushed_len(), 21); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 21); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + vec.write()?; + } + + { + let mut vec: VEC = PcoVec::forced_import_with(options)?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.len(), 21); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(10, 11), vec![10]); + + vec.write()?; + } + + { + let vec: VEC = PcoVec::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + } + + Ok(()) +} diff --git a/crates/vecdb/examples/signal_test.rs b/crates/vecdb/examples/signal_test.rs new file mode 100644 index 000000000..0694d61f4 --- /dev/null +++ b/crates/vecdb/examples/signal_test.rs @@ -0,0 +1,17 @@ +use std::{thread, time::Duration}; +use vecdb::Exit; + +fn main() { + let exit = Exit::new(); + exit.register_cleanup(|| { + eprintln!("[cleanup] flushing data..."); + }); + exit.set_ctrlc_handler(); + + eprintln!("Running... press Ctrl+C to test signal handling"); + for i in 1.. { + let _lock = exit.lock(); + eprintln!(" tick {i}"); + thread::sleep(Duration::from_secs(1)); + } +} diff --git a/crates/vecdb/examples/zerocopy.rs b/crates/vecdb/examples/zerocopy.rs new file mode 100644 index 000000000..327068e21 --- /dev/null +++ b/crates/vecdb/examples/zerocopy.rs @@ -0,0 +1,839 @@ +use std::{collections::BTreeSet, fs, path::Path}; + +use vecdb::{ + AnyStoredVec, AnyVec, Database, ImportableVec, ReadableVec, Stamp, Version, WritableVec, + ZeroCopyVec, +}; + +#[allow(clippy::upper_case_acronyms)] +type VEC = ZeroCopyVec; + +fn main() -> Result<(), Box> { + let _ = fs::remove_dir_all("raw"); + + let version = Version::TWO; + + let database = Database::open(Path::new("raw"))?; + + let mut options = (&database, "vec", version).into(); + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + vec.write()?; + + assert!(vec.header().stamp() == Stamp::new(0)); + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + vec.mut_header().update_stamp(Stamp::new(100)); + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.collect_range(0, 6), vec![0, 1, 2, 3, 4, 5]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + + vec.push(21); + vec.push(22); + + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.pushed_len(), 2); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(20, 23), vec![20, 21, 22]); + assert!(vec.collect_range(23, 24).is_empty()); + + vec.write()?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.stored_len(), 23); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 23), vec![20, 21, 22]); + + vec.truncate_if_needed(14)?; + + assert_eq!(vec.stored_len(), 14); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 14); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert!(vec.collect_range(20, 21).is_empty()); + + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![9, 10, 11, 12, 13] + ); + + vec.push(vec.len() as u32); + let all = vec.collect(); + assert_eq!(*all.last().unwrap(), 14); + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + + vec.write()?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + let all = vec.collect(); + assert_eq!(*all.last().unwrap(), 14); + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + + vec.reset()?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 0); + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.pushed_len(), 21); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 21); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + let reader = vec.reader(); + assert_eq!(vec.take(10, &reader), Some(10)); + assert_eq!(vec.holes(), &BTreeSet::from([10])); + assert_eq!(vec.get_with_reader(10, &reader), None); + drop(reader); + + vec.write()?; + + assert!(vec.holes() == &BTreeSet::from([10])); + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert!(vec.holes() == &BTreeSet::from([10])); + + let reader = vec.reader(); + assert!(vec.get_with_reader(10, &reader).is_none()); + drop(reader); + + vec.update(10, 10)?; + vec.update(0, 10)?; + + let reader = vec.reader(); + assert_eq!(vec.holes(), &BTreeSet::new()); + assert_eq!(vec.get_with_reader(0, &reader), Some(10)); + assert_eq!(vec.get_with_reader(10, &reader), Some(10)); + drop(reader); + + vec.write()?; + } + + options = options.with_saved_stamped_changes(10); + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + vec.truncate_if_needed(10)?; + + let reader = vec.reader(); + let _ = vec.take(5, &reader); + vec.update(3, 5)?; + vec.push(21); + drop(reader); + + assert_eq!( + vec.collect_holed(), + vec![ + Some(10), + Some(1), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(1))?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert_eq!(vec.collect(), vec![10, 1, 2, 5, 4, 6, 7, 8, 9, 21]); + + let reader = vec.reader(); + let _ = vec.take(0, &reader); + vec.update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(2))?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + vec.rollback()?; + + assert_eq!(vec.stamp(), Stamp::new(1)); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect_holed(), + vec![ + Some(10), + Some(1), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21) + ] + ); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + vec.rollback()?; + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.stamped_write(Stamp::new(0))?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.truncate_if_needed(10)?; + + let reader = vec.reader(); + let _ = vec.take(5, &reader); + vec.update(3, 5)?; + vec.push(21); + drop(reader); + + assert_eq!( + vec.collect_holed(), + vec![ + Some(10), + Some(1), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(1))?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert_eq!(vec.collect(), vec![10, 1, 2, 5, 4, 6, 7, 8, 9, 21]); + + let reader = vec.reader(); + let _ = vec.take(0, &reader); + vec.update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(2))?; + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + let _ = vec.rollback_before(Stamp::new(1))?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.stamped_write(Stamp::new(0))?; + + vec.truncate_if_needed(10)?; + let reader = vec.reader(); + let _ = vec.take(5, &reader); + vec.update(3, 5)?; + vec.push(21); + drop(reader); + + let reader = vec.reader(); + let _ = vec.take(0, &reader); + vec.update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + } + + { + let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + dbg!(("0", vec.prev_holes(), vec.updated())); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.truncate_if_needed(10)?; + let reader = vec.reader(); + let _ = vec.take(5, &reader); + vec.update(3, 5)?; + vec.push(21); + drop(reader); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.stamp(), Stamp::new(1)); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + let reader = vec.reader(); + let _ = vec.take(0, &reader); + vec.update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + vec.stamped_write_with_changes(Stamp::new(2))?; + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + let _ = vec.rollback_before(Stamp::new(1))?; + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + assert_eq!(vec.stamp(), Stamp::new(0)); + + vec.truncate_if_needed(10)?; + let reader = vec.reader(); + let _ = vec.take(5, &reader); + vec.update(3, 5)?; + vec.push(21); + drop(reader); + + let reader = vec.reader(); + let _ = vec.take(0, &reader); + vec.update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!( + vec.collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + assert_eq!(vec.stamp(), Stamp::new(0)); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.stamp(), Stamp::new(2)); + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + dbg!("-----------------------------------------------------------------------"); + + let _ = vec.rollback_before(Stamp::new(1))?; + + dbg!(( + vec.stored_len(), + vec.real_stored_len(), + vec.pushed(), + vec.prev_updated(), + vec.updated(), + vec.prev_holes(), + vec.holes(), + )); + + assert_eq!(vec.stamp(), Stamp::new(0)); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.stamped_write_with_changes(Stamp::new(0))?; + + let vec: VEC = ZeroCopyVec::forced_import_with(options)?; + dbg!(("0", vec.prev_holes(), vec.updated())); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + } + + // { + // let mut vec: VEC = ZeroCopyVec::forced_import_with(options)?; + + // dbg!(("0", vec.stamp(), vec.stored_len())); + + // vec.truncate_if_needed(10)?; + // let reader = vec.reader(); + // vec.take(5, &reader); + // vec.update(3, 5)?; + // vec.push(21); + // drop(reader); + + // vec.stamped_write_with_changes(Stamp::new(1))?; + // assert_eq!(vec.stamp(), Stamp::new(1)); + + // dbg!(("1", vec.stamp(), vec.stored_len())); + + // let reader = vec.reader(); + // vec.take(0, &reader); + // vec.update(1, 5)?; + // vec.push(5); + // vec.push(6); + // vec.push(7); + // drop(reader); + + // vec.stamped_write_with_changes(Stamp::new(2))?; + + // assert_eq!( + // vec.collect_holed(), + // vec![ + // None, + // Some(5), + // Some(2), + // Some(5), + // Some(4), + // None, + // Some(6), + // Some(7), + // Some(8), + // Some(9), + // Some(21), + // Some(5), + // Some(6), + // Some(7) + // ] + // ); + + // vec.rollback_before(Stamp::new(1))?; + // dbg!(("roll", vec.stamp(), vec.stored_len())); + // assert_eq!(vec.stamp(), Stamp::new(0)); + + // vec.truncate_if_needed(10)?; + // let reader = vec.reader(); + // vec.take(5, &reader); + // vec.update(3, 5)?; + // vec.push(21); + // drop(reader); + + // let reader = vec.reader(); + // vec.take(0, &reader); + // vec.update(1, 5)?; + // vec.push(5); + // vec.push(6); + // vec.push(7); + // drop(reader); + + // assert_eq!( + // vec.collect_holed(), + // vec![ + // None, + // Some(5), + // Some(2), + // Some(5), + // Some(4), + // None, + // Some(6), + // Some(7), + // Some(8), + // Some(9), + // Some(21), + // Some(5), + // Some(6), + // Some(7) + // ] + // ); + + // dbg!(1); + + // vec.rollback_before(Stamp::new(1))?; + // dbg!(1); + + // assert_eq!(vec.stamp(), Stamp::new(0)); + + // assert_eq!( + // vec.collect(), + // vec![ + // 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + // ] + // ); + + // vec.stamped_write_with_changes(Stamp::new(0))?; + + // let vec: VEC = ZeroCopyVec::forced_import_with(options)?; + // dbg!(("0", vec.prev_holes(), vec.updated())); + + // assert_eq!( + // vec.collect(), + // vec![ + // 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + // ] + // ); + // } + + Ok(()) +} + +// fn main() {} diff --git a/crates/vecdb/src/base/change/cursor.rs b/crates/vecdb/src/base/change/cursor.rs new file mode 100644 index 000000000..00b2d3db5 --- /dev/null +++ b/crates/vecdb/src/base/change/cursor.rs @@ -0,0 +1,60 @@ +use crate::{Bytes, Error, Result, SIZE_OF_U64, Stamp}; + +/// Position-tracking reader for change-file payloads. +pub(crate) struct ChangeCursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> ChangeCursor<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } + + pub fn read_u64(&mut self) -> Result { + self.check_remaining(SIZE_OF_U64)?; + let v = usize::from_bytes(&self.bytes[self.pos..self.pos + SIZE_OF_U64])?; + self.pos += SIZE_OF_U64; + Ok(v) + } + + pub fn read_stamp(&mut self) -> Result { + self.check_remaining(SIZE_OF_U64)?; + let v = Stamp::from_bytes(&self.bytes[self.pos..self.pos + SIZE_OF_U64])?; + self.pos += SIZE_OF_U64; + Ok(v) + } + + pub fn skip(&mut self, n: usize) -> Result<()> { + self.check_remaining(n)?; + self.pos += n; + Ok(()) + } + + pub fn read_values Result>( + &mut self, + count: usize, + size_of_t: usize, + mut read: F, + ) -> Result> { + let total = size_of_t.checked_mul(count).ok_or(Error::Overflow)?; + self.check_remaining(total)?; + let vals = self.bytes[self.pos..self.pos + total] + .chunks(size_of_t) + .map(&mut read) + .collect::>>()?; + self.pos += total; + Ok(vals) + } + + fn check_remaining(&self, len: usize) -> Result<()> { + let end = self.pos.checked_add(len).ok_or(Error::Overflow)?; + if end > self.bytes.len() { + return Err(Error::WrongLength { + received: self.bytes.len(), + expected: end, + }); + } + Ok(()) + } +} diff --git a/crates/vecdb/src/base/change/data.rs b/crates/vecdb/src/base/change/data.rs new file mode 100644 index 000000000..ea4f079d2 --- /dev/null +++ b/crates/vecdb/src/base/change/data.rs @@ -0,0 +1,11 @@ +use crate::Stamp; + +/// Parsed change data returned by parse_change_data, consumed by apply_rollback. +#[derive(Debug)] +pub(crate) struct ChangeData { + pub prev_stamp: Stamp, + pub prev_stored_len: usize, + pub truncated_start: usize, + pub truncated_values: Vec, + pub prev_pushed: Vec, +} diff --git a/crates/vecdb/src/base/change/mod.rs b/crates/vecdb/src/base/change/mod.rs new file mode 100644 index 000000000..010bc50d6 --- /dev/null +++ b/crates/vecdb/src/base/change/mod.rs @@ -0,0 +1,5 @@ +mod cursor; +mod data; + +pub(crate) use cursor::ChangeCursor; +pub(crate) use data::ChangeData; diff --git a/crates/vecdb/src/base/format/bytes.rs b/crates/vecdb/src/base/format/bytes.rs new file mode 100644 index 000000000..3534058f6 --- /dev/null +++ b/crates/vecdb/src/base/format/bytes.rs @@ -0,0 +1,32 @@ +use crate::{Bytes, Error, Result}; + +use super::Format; + +impl Bytes for Format { + type Array = [u8; size_of::()]; + + #[inline] + fn to_bytes(&self) -> Self::Array { + [*self as u8] + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> Result { + let len = bytes.len(); + if len != size_of::() { + return Err(Error::WrongLength { + expected: size_of::(), + received: len, + }); + } + + match bytes[0] { + 0 => Ok(Self::Bytes), + 1 => Ok(Self::ZeroCopy), + 64 => Ok(Self::Pco), + 65 => Ok(Self::LZ4), + 66 => Ok(Self::Zstd), + b => Err(Error::InvalidFormat(b)), + } + } +} diff --git a/crates/vecdb/src/base/format/mod.rs b/crates/vecdb/src/base/format/mod.rs new file mode 100644 index 000000000..4a609075e --- /dev/null +++ b/crates/vecdb/src/base/format/mod.rs @@ -0,0 +1,55 @@ +mod bytes; + +/// Storage format selection for stored vectors. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Format { + /// Explicit byte serialization with little-endian byte order. + /// **PORTABLE** across different endianness systems. Uses custom Bytes trait. + Bytes, + /// Direct memory mapping with native byte order via zerocopy. + /// **NOT PORTABLE** - fastest but endianness-dependent. Best for random access. + ZeroCopy, + /// Pcodec compression optimized for numeric sequences (best compression for numbers). + Pco = 64, + /// LZ4 compression (fastest compression/decompression, moderate ratio). + LZ4 = 65, + /// Zstd compression (highest compression ratio, slower). + Zstd = 66, +} + +impl Format { + #[inline] + pub fn is_raw(&self) -> bool { + matches!(self, Self::ZeroCopy | Self::Bytes) + } + + #[inline] + pub fn is_compressed(&self) -> bool { + matches!(self, Self::Pco | Self::LZ4 | Self::Zstd) + } + + #[inline] + pub fn is_zerocopy(&self) -> bool { + *self == Self::ZeroCopy + } + + #[inline] + pub fn is_bytes(&self) -> bool { + *self == Self::Bytes + } + + #[inline] + pub fn is_pcodec(&self) -> bool { + *self == Self::Pco + } + + #[inline] + pub fn is_lz4(&self) -> bool { + *self == Self::LZ4 + } + + #[inline] + pub fn is_zstd(&self) -> bool { + *self == Self::Zstd + } +} diff --git a/crates/vecdb/src/base/header/inner.rs b/crates/vecdb/src/base/header/inner.rs new file mode 100644 index 000000000..88669ad18 --- /dev/null +++ b/crates/vecdb/src/base/header/inner.rs @@ -0,0 +1,118 @@ +use rawdb::Region; + +use crate::{Bytes, Error, Result, Stamp, Version}; + +use super::{super::Format, HEADER_OFFSET, HEADER_VERSION}; + +#[derive(Debug, Clone)] +#[repr(C)] +pub(super) struct HeaderInner { + pub header_version: Version, + pub vec_version: Version, + pub computed_version: Version, + pub stamp: Stamp, + pub format: Format, +} + +impl HeaderInner { + pub fn create_and_write(region: &Region, vec_version: Version, format: Format) -> Result { + let header = Self { + header_version: HEADER_VERSION, + vec_version, + computed_version: Version::default(), + stamp: Stamp::default(), + format, + }; + header.write(region)?; + Ok(header) + } + + pub fn write(&self, region: &Region) -> Result<()> { + region.write_at(&self.to_bytes(), 0)?; + Ok(()) + } + + pub fn import_and_verify( + region: &Region, + vec_version: Version, + format: Format, + ) -> Result { + let len = region.meta().len(); + + if len < HEADER_OFFSET { + return Err(Error::WrongLength { + expected: HEADER_OFFSET, + received: len, + }); + } + + let reader = region.create_reader(); + let vec = reader.unchecked_read(0, HEADER_OFFSET); + let header = HeaderInner::from_bytes(vec)?; + + if header.header_version != HEADER_VERSION { + return Err(Error::DifferentVersion { + received: header.header_version, + expected: HEADER_VERSION, + }); + } + if header.vec_version != vec_version { + return Err(Error::DifferentVersion { + received: header.vec_version, + expected: vec_version, + }); + } + + if header.format != format { + return Err(Error::DifferentFormat { + received: header.format, + expected: format, + }); + } + + Ok(header) + } + + fn to_bytes(&self) -> [u8; HEADER_OFFSET] { + let mut buf = [0u8; HEADER_OFFSET]; + let mut pos = 0; + let hv = self.header_version.to_bytes(); + buf[pos..pos + hv.len()].copy_from_slice(&hv); + pos += hv.len(); + let vv = self.vec_version.to_bytes(); + buf[pos..pos + vv.len()].copy_from_slice(&vv); + pos += vv.len(); + let cv = self.computed_version.to_bytes(); + buf[pos..pos + cv.len()].copy_from_slice(&cv); + pos += cv.len(); + let s = self.stamp.to_bytes(); + buf[pos..pos + s.len()].copy_from_slice(&s); + pos += s.len(); + let f = self.format.to_bytes(); + buf[pos..pos + f.len()].copy_from_slice(&f); + // remaining bytes are already zero (padding) + buf + } + + fn from_bytes(bytes: &[u8]) -> Result { + let len = bytes.len(); + if len < HEADER_OFFSET { + return Err(Error::WrongLength { + expected: HEADER_OFFSET, + received: len, + }); + } + let header_version = Version::from_bytes(&bytes[0..4])?; + let vec_version = Version::from_bytes(&bytes[4..8])?; + let computed_version = Version::from_bytes(&bytes[8..12])?; + let stamp = Stamp::from_bytes(&bytes[12..20])?; + let format = Format::from_bytes(&bytes[20..21])?; + Ok(Self { + header_version, + vec_version, + computed_version, + stamp, + format, + }) + } +} diff --git a/crates/vecdb/src/base/header/mod.rs b/crates/vecdb/src/base/header/mod.rs new file mode 100644 index 000000000..0080f9446 --- /dev/null +++ b/crates/vecdb/src/base/header/mod.rs @@ -0,0 +1,85 @@ +use std::sync::Arc; + +use parking_lot::RwLock; +use rawdb::Region; + +mod inner; + +use inner::HeaderInner; + +use crate::{Result, Stamp, Version}; + +use super::Format; + +const HEADER_VERSION: Version = Version::TWO; +pub const HEADER_OFFSET: usize = size_of::(); + +#[derive(Debug, Clone)] +pub struct Header { + inner: Arc>, + modified: bool, +} + +impl Header { + pub fn create_and_write(region: &Region, vec_version: Version, format: Format) -> Result { + let inner = HeaderInner::create_and_write(region, vec_version, format)?; + Ok(Self { + inner: Arc::new(RwLock::new(inner)), + modified: false, + }) + } + + pub fn import_and_verify( + region: &Region, + vec_version: Version, + format: Format, + ) -> Result { + let inner = HeaderInner::import_and_verify(region, vec_version, format)?; + Ok(Self { + inner: Arc::new(RwLock::new(inner)), + modified: false, + }) + } + + pub fn update_stamp(&mut self, stamp: Stamp) { + let mut inner = self.inner.write(); + if inner.stamp != stamp { + self.modified = true; + inner.stamp = stamp; + } + } + + pub fn update_computed_version(&mut self, computed_version: Version) { + let mut inner = self.inner.write(); + if inner.computed_version != computed_version { + self.modified = true; + inner.computed_version = computed_version; + } + } + + #[inline(always)] + pub fn modified(&self) -> bool { + self.modified + } + + #[inline(always)] + pub fn vec_version(&self) -> Version { + self.inner.read().vec_version + } + + #[inline(always)] + pub fn computed_version(&self) -> Version { + self.inner.read().computed_version + } + + #[inline(always)] + pub fn stamp(&self) -> Stamp { + self.inner.read().stamp + } + + pub fn write(&mut self, region: &Region) -> Result<()> { + self.inner.read().write(region)?; + self.modified = false; + Ok(()) + } +} diff --git a/crates/vecdb/src/base/mod.rs b/crates/vecdb/src/base/mod.rs new file mode 100644 index 000000000..c90b5811a --- /dev/null +++ b/crates/vecdb/src/base/mod.rs @@ -0,0 +1,20 @@ +//! Shared storage substrate used by every concrete vec variant. + +mod change; +mod format; +mod header; +mod options; +mod read_only; +mod read_write; +mod rollback; +mod shared_len; +mod with_prev; + +pub(crate) use change::*; +pub use format::*; +pub use header::*; +pub use options::*; +pub(crate) use read_only::*; +pub use read_write::*; +pub use shared_len::*; +pub use with_prev::*; diff --git a/crates/vecdb/src/base/options/from.rs b/crates/vecdb/src/base/options/from.rs new file mode 100644 index 000000000..90a10615d --- /dev/null +++ b/crates/vecdb/src/base/options/from.rs @@ -0,0 +1,11 @@ +use rawdb::Database; + +use crate::Version; + +use super::ImportOptions; + +impl<'a> From<(&'a Database, &'a str, Version)> for ImportOptions<'a> { + fn from((db, name, version): (&'a Database, &'a str, Version)) -> Self { + Self::new(db, name, version) + } +} diff --git a/crates/vecdb/src/base/options/mod.rs b/crates/vecdb/src/base/options/mod.rs new file mode 100644 index 000000000..fd84350f3 --- /dev/null +++ b/crates/vecdb/src/base/options/mod.rs @@ -0,0 +1,34 @@ +use rawdb::Database; + +mod from; + +use crate::Version; + +/// Options for importing or creating stored vectors. +#[derive(Clone, Copy)] +pub struct ImportOptions<'a> { + /// Database to store the vector in. + pub db: &'a Database, + /// Name of the vector. + pub name: &'a str, + /// Version for tracking data schema compatibility. + pub version: Version, + /// Number of stamped change files to keep for rollback support (0 to disable). + pub saved_stamped_changes: u16, +} + +impl<'a> ImportOptions<'a> { + pub fn new(db: &'a Database, name: &'a str, version: Version) -> Self { + Self { + db, + name, + version, + saved_stamped_changes: 0, + } + } + + pub fn with_saved_stamped_changes(mut self, num: u16) -> Self { + self.saved_stamped_changes = num; + self + } +} diff --git a/crates/vecdb/src/base/read_only.rs b/crates/vecdb/src/base/read_only.rs new file mode 100644 index 000000000..f572c0003 --- /dev/null +++ b/crates/vecdb/src/base/read_only.rs @@ -0,0 +1,57 @@ +use std::{marker::PhantomData, sync::Arc}; + +use rawdb::Region; + +use crate::{VecIndex, VecValue, Version}; + +use super::{Header, SharedLen}; + +/// Read-only core of a stored vector — the minimal state needed for disk reads. +/// +/// Contains region (I/O), shared length (bounds), name/header (metadata). +/// No pushed buffers, no rollback state. +#[derive(Debug, Clone)] +pub(crate) struct ReadOnlyBaseVec { + pub(crate) region: Region, + pub(crate) stored_len: SharedLen, + pub(crate) name: Arc, + pub(crate) header: Header, + pub(crate) phantom: PhantomData<(I, T)>, +} + +impl ReadOnlyBaseVec +where + I: VecIndex, + T: VecValue, +{ + #[inline(always)] + pub fn region(&self) -> &Region { + &self.region + } + + #[inline(always)] + pub fn header(&self) -> &Header { + &self.header + } + + #[inline(always)] + pub fn name(&self) -> &str { + &self.name + } + + #[inline(always)] + pub fn stored_len(&self) -> usize { + self.stored_len.get() + } + + /// For read-only vecs, len == stored_len (no pushed buffer). + #[inline(always)] + pub fn len(&self) -> usize { + self.stored_len.get() + } + + #[inline(always)] + pub fn version(&self) -> Version { + self.header.vec_version() + } +} diff --git a/crates/vecdb/src/base/read_write.rs b/crates/vecdb/src/base/read_write.rs new file mode 100644 index 000000000..b90c420ec --- /dev/null +++ b/crates/vecdb/src/base/read_write.rs @@ -0,0 +1,241 @@ +use std::{fs, marker::PhantomData, ops::Deref, path::PathBuf, sync::Arc}; + +use rawdb::Database; + +use crate::{Error, Result, Stamp, VecIndex, VecValue}; + +use super::{Format, HEADER_OFFSET, Header, ImportOptions, ReadOnlyBaseVec, SharedLen, WithPrev}; + +/// Base storage vector with fields common to all stored vector implementations. +/// +/// Derefs to [`ReadOnlyBaseVec`] for read-only access to region, header, name, +/// stored_len, and version. Write state (pushed, rollback) lives here. +#[derive(Debug)] +pub(crate) struct ReadWriteBaseVec { + pub(crate) read_only: ReadOnlyBaseVec, + pub(super) pushed: WithPrev>, + pub(super) previous_stored_len: usize, + pub(super) saved_stamped_changes: u16, +} + +impl Deref for ReadWriteBaseVec { + type Target = ReadOnlyBaseVec; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.read_only + } +} + +impl ReadWriteBaseVec +where + I: VecIndex, + T: VecValue, +{ + pub fn import(options: ImportOptions, format: Format) -> Result { + let region = options + .db + .create_region_if_needed(&vec_region_name_with::(options.name))?; + + let region_len = region.meta().len(); + if region_len > 0 && region_len < HEADER_OFFSET { + return Err(Error::CorruptedRegion { + name: region.meta().id().to_string(), + region_len, + }); + } + + let header = if region_len == 0 { + Header::create_and_write(®ion, options.version, format)? + } else { + Header::import_and_verify(®ion, options.version, format)? + }; + + Ok(Self { + read_only: ReadOnlyBaseVec { + region, + header, + name: Arc::from(options.name), + stored_len: SharedLen::default(), + phantom: PhantomData, + }, + pushed: WithPrev::default(), + previous_stored_len: 0, + saved_stamped_changes: options.saved_stamped_changes, + }) + } + + #[inline] + pub fn read_only_base(&self) -> ReadOnlyBaseVec { + self.read_only.clone() + } + + #[inline] + pub fn pushed(&self) -> &[T] { + self.pushed.current() + } + + #[inline] + pub fn mut_pushed(&mut self) -> &mut Vec { + self.pushed.current_mut() + } + + #[inline] + pub fn reserve_pushed(&mut self, additional: usize) { + self.pushed.current_mut().reserve(additional); + } + + #[inline] + pub fn prev_pushed(&self) -> &[T] { + self.pushed.previous() + } + + #[inline] + pub fn len(&self) -> usize { + self.stored_len() + self.pushed().len() + } + + #[inline] + pub fn update_stored_len(&self, val: usize) { + self.read_only.stored_len.set(val); + } + + #[inline] + pub fn prev_stored_len(&self) -> usize { + self.previous_stored_len + } + + #[inline(always)] + pub fn mut_prev_stored_len(&mut self) -> &mut usize { + &mut self.previous_stored_len + } + + #[inline(always)] + pub fn saved_stamped_changes(&self) -> u16 { + self.saved_stamped_changes + } + + #[inline] + pub fn mut_header(&mut self) -> &mut Header { + &mut self.read_only.header + } + + #[inline] + pub fn db(&self) -> Database { + self.region.db() + } + + #[inline] + pub fn db_path(&self) -> PathBuf { + self.db().path().to_path_buf() + } + + pub fn write_header_if_needed(&mut self) -> Result<()> { + if self.read_only.header.modified() { + self.read_only.header.write(&self.read_only.region)?; + } + Ok(()) + } + + pub fn index_to_name(&self) -> String { + vec_region_name(&self.name, I::to_string()) + } + + pub fn remove(self) -> Result<()> { + self.read_only.region.remove()?; + Ok(()) + } + + /// Tight pointer loop for LLVM vectorization. + #[inline] + pub fn fold_pushed B>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> B { + let stored_len = self.stored_len(); + let start = from.max(stored_len); + if start >= to { + return init; + } + let pushed = self.pushed(); + let slice_from = start - stored_len; + let slice_to = (to - stored_len).min(pushed.len()); + let mut acc = init; + let mut i = slice_from; + while i < slice_to { + acc = f(acc, pushed[i].clone()); + i += 1; + } + acc + } + + #[inline] + pub fn try_fold_pushed std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result { + let stored_len = self.stored_len(); + let start = from.max(stored_len); + if start >= to { + return Ok(init); + } + let pushed = self.pushed(); + let mut acc = init; + for v in &pushed[(start - stored_len)..(to - stored_len).min(pushed.len())] { + acc = f(acc, v.clone())?; + } + Ok(acc) + } + + /// Returns true if stored_len needs updating to `index`. + pub fn truncate_pushed(&mut self, index: usize) -> bool { + let stored_len = self.stored_len(); + let len = stored_len + self.pushed().len(); + + if index >= len { + return false; + } + + if index <= stored_len { + self.pushed.current_mut().clear(); + } else { + self.pushed.current_mut().truncate(index - stored_len); + } + + index < stored_len + } + + pub fn reset_base(&mut self) -> Result<()> { + self.pushed.clear(); + self.read_only.stored_len.set(0); + self.previous_stored_len = 0; + self.read_only.header.update_stamp(Stamp::default()); + + let changes_path = self.changes_path(); + if changes_path.exists() { + fs::remove_dir_all(&changes_path)?; + } + + Ok(()) + } + + pub fn reset_unsaved_base(&mut self) { + self.pushed.current_mut().clear(); + } +} + +/// Region name for a vec of type `I`, e.g. `"height/Height"`. +pub fn vec_region_name_with(name: &str) -> String { + vec_region_name(name, I::to_string()) +} + +/// Formats a vec's region name as `{name}/{index}`. +pub fn vec_region_name(name: &str, index: &str) -> String { + format!("{name}/{index}") +} diff --git a/crates/vecdb/src/base/rollback.rs b/crates/vecdb/src/base/rollback.rs new file mode 100644 index 000000000..3860fc241 --- /dev/null +++ b/crates/vecdb/src/base/rollback.rs @@ -0,0 +1,149 @@ +use std::{collections::BTreeMap, fs, path::PathBuf}; + +use crate::{Bytes, Error, Result, SIZE_OF_U64, Stamp, VecIndex, VecValue}; + +use super::{ChangeCursor, ChangeData, ReadWriteBaseVec, vec_region_name}; + +impl ReadWriteBaseVec +where + I: VecIndex, + T: VecValue, +{ + pub fn changes_path(&self) -> PathBuf { + self.db_path() + .join("changes") + .join(vec_region_name(&self.name, I::to_string())) + } + + pub fn serialize_changes( + &self, + size_of_t: usize, + collect_stored: impl FnOnce(usize, usize) -> Result>, + write_values: impl Fn(&[T], &mut Vec), + ) -> Result> { + let prev_stored_len = self.prev_stored_len(); + let stored_len = self.stored_len(); + let truncated = prev_stored_len.saturating_sub(stored_len); + + let value_count = truncated + self.prev_pushed().len() + self.pushed().len(); + let mut bytes = Vec::with_capacity(6 * SIZE_OF_U64 + value_count * size_of_t); + + bytes.extend(self.header.stamp().to_bytes()); + bytes.extend(prev_stored_len.to_bytes()); + bytes.extend(stored_len.to_bytes()); + bytes.extend(truncated.to_bytes()); + + if truncated > 0 { + let truncated_vals = collect_stored(stored_len, prev_stored_len)?; + write_values(&truncated_vals, &mut bytes); + } + + bytes.extend(self.prev_pushed().len().to_bytes()); + write_values(self.prev_pushed(), &mut bytes); + + bytes.extend(self.pushed().len().to_bytes()); + write_values(self.pushed(), &mut bytes); + + Ok(bytes) + } + + /// Returns `Error::Overflow` on arithmetic overflow, + /// `Error::WrongLength` if the data is truncated. + pub fn parse_change_data( + c: &mut ChangeCursor, + size_of_t: usize, + read_value: impl Fn(&[u8]) -> Result, + ) -> Result> { + let prev_stamp = c.read_stamp()?; + let prev_stored_len = c.read_u64()?; + c.skip(SIZE_OF_U64)?; // stored_len, not needed for rollback + let truncated_count = c.read_u64()?; + + let truncated_start = prev_stored_len + .checked_sub(truncated_count) + .ok_or(Error::Underflow)?; + let truncated_values = c.read_values(truncated_count, size_of_t, &read_value)?; + + let prev_pushed_len = c.read_u64()?; + let prev_pushed = c.read_values(prev_pushed_len, size_of_t, &read_value)?; + + let pushed_len = c.read_u64()?; + c.skip(size_of_t.checked_mul(pushed_len).ok_or(Error::Overflow)?)?; + + Ok(ChangeData { + prev_stamp, + prev_stored_len, + truncated_start, + truncated_values, + prev_pushed, + }) + } + + /// Restores the base rollback state. Caller resolves `stored_len` and + /// `pushed` from the parsed change data according to its own overlay + /// strategy (raw uses an `updated` map for truncated values, compressed + /// re-queues them in `pushed`). + pub fn apply_rollback(&mut self, stamp: Stamp, stored_len: usize, pushed: Vec) { + self.read_only.header.update_stamp(stamp); + self.read_only.stored_len.set(stored_len); + *self.pushed.current_mut() = pushed; + self.pushed.save(); + } + + /// Caller must check `saved_stamped_changes > 0` before calling. + pub fn save_change_file(&self, stamp: Stamp, data: &[u8]) -> Result<()> { + debug_assert!(self.saved_stamped_changes > 0); + let path = self.changes_path(); + fs::create_dir_all(&path)?; + + let files: BTreeMap = fs::read_dir(&path)? + .filter_map(|entry| { + let path = entry.ok()?.path(); + let s = Stamp::from(path.file_name()?.to_str()?.parse::().ok()?); + if s < stamp { + Some((s, path)) + } else { + let _ = fs::remove_file(&path); + None + } + }) + .collect(); + + let excess = files + .len() + .saturating_sub(self.saved_stamped_changes as usize - 1); + for (_, path) in files.iter().take(excess) { + fs::remove_file(path)?; + } + + fs::write(path.join(u64::from(stamp).to_string()), data)?; + Ok(()) + } + + pub fn save_prev(&mut self) { + self.previous_stored_len = self.stored_len(); + self.pushed.previous_mut().clear(); + } + + pub fn save_prev_for_rollback(&mut self) { + self.previous_stored_len = self.stored_len(); + self.pushed.save(); + } + + pub fn read_current_change_file(&self) -> Result> { + let path = self + .changes_path() + .join(u64::from(self.header.stamp()).to_string()); + Ok(fs::read(path)?) + } + + pub fn find_rollback_files(&self) -> Result> { + Ok(fs::read_dir(self.changes_path())? + .filter_map(|entry| { + let path = entry.ok()?.path(); + let stamp = Stamp::from(path.file_name()?.to_str()?.parse::().ok()?); + Some((stamp, path)) + }) + .collect()) + } +} diff --git a/crates/vecdb/src/base/shared_len/default.rs b/crates/vecdb/src/base/shared_len/default.rs new file mode 100644 index 000000000..7f5faf692 --- /dev/null +++ b/crates/vecdb/src/base/shared_len/default.rs @@ -0,0 +1,7 @@ +use super::SharedLen; + +impl Default for SharedLen { + fn default() -> Self { + Self::new(0) + } +} diff --git a/crates/vecdb/src/base/shared_len/mod.rs b/crates/vecdb/src/base/shared_len/mod.rs new file mode 100644 index 000000000..a4ad1ff37 --- /dev/null +++ b/crates/vecdb/src/base/shared_len/mod.rs @@ -0,0 +1,32 @@ +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +mod default; + +/// Atomic length counter shared across clones. +/// +/// Wraps `Arc` to allow multiple vec instances (original and clones) +/// to observe the same length as the original grows. +#[derive(Debug, Clone)] +pub struct SharedLen(Arc); + +impl SharedLen { + /// Creates a new shared length counter. + pub fn new(val: usize) -> Self { + Self(Arc::new(AtomicUsize::new(val))) + } + + /// Gets the current length. + #[inline(always)] + pub fn get(&self) -> usize { + self.0.load(Ordering::Acquire) + } + + /// Sets the length. + #[inline] + pub fn set(&self, val: usize) { + self.0.store(val, Ordering::Release); + } +} diff --git a/crates/vecdb/src/base/with_prev/default.rs b/crates/vecdb/src/base/with_prev/default.rs new file mode 100644 index 000000000..d408fe0a8 --- /dev/null +++ b/crates/vecdb/src/base/with_prev/default.rs @@ -0,0 +1,10 @@ +use super::WithPrev; + +impl Default for WithPrev { + fn default() -> Self { + Self { + current: T::default(), + previous: T::default(), + } + } +} diff --git a/crates/vecdb/src/base/with_prev/mod.rs b/crates/vecdb/src/base/with_prev/mod.rs new file mode 100644 index 000000000..6d54a6363 --- /dev/null +++ b/crates/vecdb/src/base/with_prev/mod.rs @@ -0,0 +1,96 @@ +mod default; + +/// Tracks current and previous values for rollback support. +#[derive(Debug, Clone)] +pub struct WithPrev { + pub(super) current: T, + pub(super) previous: T, +} + +impl WithPrev { + pub fn new(value: T) -> Self + where + T: Clone, + { + Self { + current: value.clone(), + previous: value, + } + } + + #[inline(always)] + pub fn current(&self) -> &T { + &self.current + } + + #[inline] + pub fn current_mut(&mut self) -> &mut T { + &mut self.current + } + + #[inline(always)] + pub fn previous(&self) -> &T { + &self.previous + } + + #[inline] + pub fn previous_mut(&mut self) -> &mut T { + &mut self.previous + } + + /// Copies current into previous. + #[inline] + pub fn save(&mut self) + where + T: Clone, + { + self.previous.clone_from(&self.current); + } + + /// Copies previous into current. + #[inline] + pub fn restore(&mut self) + where + T: Clone, + { + self.current.clone_from(&self.previous); + } + + #[inline] + pub fn swap(&mut self) { + std::mem::swap(&mut self.current, &mut self.previous); + } + + #[inline] + pub fn take_current(&mut self) -> T + where + T: Default, + { + std::mem::take(&mut self.current) + } + + #[inline] + pub fn take_previous(&mut self) -> T + where + T: Default, + { + std::mem::take(&mut self.previous) + } + + #[inline] + pub fn clear(&mut self) + where + T: Default, + { + self.current = T::default(); + self.previous = T::default(); + } + + #[inline] + pub fn clear_previous(&mut self) + where + T: Default, + { + self.previous = T::default(); + } +} diff --git a/crates/vecdb/src/bytes/array.rs b/crates/vecdb/src/bytes/array.rs new file mode 100644 index 000000000..b9bce7a00 --- /dev/null +++ b/crates/vecdb/src/bytes/array.rs @@ -0,0 +1,32 @@ +use crate::{Error, Result}; + +use super::Bytes; + +macro_rules! impl_bytes_for_array { + ($($n:expr),*) => { + $( + impl Bytes for [u8; $n] { + type Array = [u8; $n]; + const IS_NATIVE_LAYOUT: bool = true; + + #[inline] + fn to_bytes(&self) -> Self::Array { + *self + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> Result { + bytes.try_into().map_err(|_| Error::WrongLength { + expected: $n, + received: bytes.len(), + }) + } + } + )* + }; +} + +impl_bytes_for_array!( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 64, 65 +); diff --git a/crates/vecdb/src/bytes/mod.rs b/crates/vecdb/src/bytes/mod.rs new file mode 100644 index 000000000..effb94b06 --- /dev/null +++ b/crates/vecdb/src/bytes/mod.rs @@ -0,0 +1,40 @@ +mod array; +mod numeric; + +use crate::Result; + +/// Trait for types that can be serialized to/from bytes with explicit byte order. +/// +/// This trait uses **LITTLE-ENDIAN** byte order for all numeric types, making the data +/// **portable across different endianness systems** (x86, ARM, etc.). This is the key +/// difference from `ZeroCopyVec`, which uses native byte order and is not portable. +/// +/// Use this trait when: +/// - You need cross-platform compatibility +/// - You're sharing data between systems with different endianness +/// - You need custom serialization logic +/// +/// For maximum performance on a single system, use `ZeroCopyVec` instead. +pub trait Bytes: Sized { + /// The byte array type returned by `to_bytes`. + /// For fixed-size types, this is `[u8; N]` where N is the size of the type. + type Array: AsRef<[u8]>; + + /// Whether the byte representation from `to_bytes` is identical to the + /// in-memory representation of Self. When true, bulk operations can use + /// memcpy instead of per-element deserialization. + /// + /// For numeric types, this is true on little-endian platforms (since + /// `to_bytes`/`from_bytes` use little-endian format which matches native). + const IS_NATIVE_LAYOUT: bool = false; + + /// Serializes this value to bytes. + /// + /// For numeric types, this uses little-endian byte order (via `to_le_bytes`). + fn to_bytes(&self) -> Self::Array; + + /// Deserializes a value from bytes. + /// + /// For numeric types, this uses little-endian byte order (via `from_le_bytes`). + fn from_bytes(bytes: &[u8]) -> Result; +} diff --git a/crates/vecdb/src/bytes/numeric.rs b/crates/vecdb/src/bytes/numeric.rs new file mode 100644 index 000000000..3492773be --- /dev/null +++ b/crates/vecdb/src/bytes/numeric.rs @@ -0,0 +1,34 @@ +use crate::{Error, Result}; + +use super::Bytes; + +macro_rules! impl_bytes_for_numeric { + ($($t:ty),*) => { + $( + impl Bytes for $t { + type Array = [u8; std::mem::size_of::<$t>()]; + const IS_NATIVE_LAYOUT: bool = cfg!(target_endian = "little"); + + #[inline] + fn to_bytes(&self) -> Self::Array { + self.to_le_bytes() + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> Result { + let arr: [u8; std::mem::size_of::<$t>()] = bytes + .try_into() + .map_err(|_| Error::WrongLength { + expected: std::mem::size_of::<$t>(), + received: bytes.len(), + })?; + Ok(<$t>::from_le_bytes(arr)) + } + } + )* + }; +} + +impl_bytes_for_numeric!( + u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, f32, f64 +); diff --git a/crates/vecdb/src/cursor.rs b/crates/vecdb/src/cursor.rs new file mode 100644 index 000000000..1b1056b95 --- /dev/null +++ b/crates/vecdb/src/cursor.rs @@ -0,0 +1,158 @@ +use std::marker::PhantomData; + +use crate::{ReadableVec, VecIndex, VecValue}; + +/// Buffered reader that reuses an internal buffer across chunked `read_into_at` calls. +/// +/// One allocation for the lifetime of the cursor. Ideal for sequential access patterns +/// (iterating tx-indexed vecs, computing rolling windows) where repeated `collect_one` +/// calls would decompress the same page thousands of times. +/// +/// Reads are aligned to chunk boundaries so that each underlying page is decompressed +/// at most once, even when access positions don't start at page-aligned offsets. +/// +/// # Example +/// ```ignore +/// let mut c = vec.cursor(); +/// while let Some(val) = c.next() { +/// // process val +/// } +/// ``` +pub struct Cursor< + 'a, + I: VecIndex, + T: VecValue, + V: ReadableVec + ?Sized = dyn ReadableVec, +> { + source: &'a V, + buf: Vec, + /// Absolute position of buf[0] in the source vec. + buf_start: usize, + /// Current absolute position in the source vec (used by sequential methods). + pos: usize, + chunk_size: usize, + len: usize, + _phantom: PhantomData, +} + +impl<'a, I: VecIndex, T: VecValue, V: ReadableVec + ?Sized> Cursor<'a, I, T, V> { + /// Creates a new cursor with the source's preferred chunk size. + #[inline] + pub fn new(source: &'a V) -> Self { + let len = source.len(); + let chunk_size = source.cursor_chunk_size().max(1); + Self { + source, + buf: Vec::with_capacity(chunk_size.min(len)), + buf_start: 0, + pos: 0, + chunk_size, + len, + _phantom: PhantomData, + } + } + + /// Returns the current absolute position. + #[inline] + pub fn position(&self) -> usize { + self.pos + } + + /// Returns the number of elements remaining. + #[inline] + pub fn remaining(&self) -> usize { + self.len.saturating_sub(self.pos) + } + + /// Advances the position by `n` without reading. Cheap — no decompression. + #[inline] + pub fn advance(&mut self, n: usize) { + self.pos = self.pos.saturating_add(n).min(self.len); + } + + /// Returns the value at absolute `index`, using the buffer when possible. + /// + /// If `index` falls within the currently buffered chunk, returns instantly. + /// Otherwise decompresses the aligned chunk containing `index`. + /// Does **not** modify the sequential position used by [`next`](Self::next). + #[inline] + pub fn get(&mut self, index: usize) -> Option { + if index >= self.len { + return None; + } + let local = self.ensure_buffered_at(index)?; + Some(self.buf[local].clone()) + } + + /// Returns the next value and advances position, or `None` if exhausted. + #[inline] + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Option { + let local = self.ensure_buffered_at(self.pos)?; + let val = self.buf[local].clone(); + self.pos += 1; + Some(val) + } + + /// Folds over the next `n` elements with a monomorphized closure. + /// Advances position by the number of elements consumed. + /// + /// Consumes any already-buffered data first, then reads fresh chunks. + /// The last chunk may read past `n` — leftover data stays in the buffer + /// for subsequent `next()` calls. + #[inline] + pub fn fold(&mut self, n: usize, init: B, mut f: impl FnMut(B, T) -> B) -> B { + let target = self.pos.saturating_add(n).min(self.len); + let mut acc = init; + + while self.pos < target { + if self.ensure_buffered_at(self.pos).is_none() { + break; + } + let local = self.pos - self.buf_start; + let local_end = (target - self.buf_start).min(self.buf.len()); + for val in self.buf[local..local_end].iter().cloned() { + acc = f(acc, val); + } + self.pos = self.buf_start + local_end; + } + + acc + } + + /// Calls `f` for each of the next `n` elements. + /// Advances position by the number of elements consumed. + #[inline] + pub fn for_each(&mut self, n: usize, mut f: impl FnMut(T)) { + self.fold(n, (), |(), v| f(v)); + } + + /// Ensures the buffer contains data at `at`. + /// Reads are aligned to `chunk_size` boundaries so that each underlying + /// compressed page is decompressed at most once across sequential accesses. + /// Returns the local index within `buf`, or `None` if out of bounds. + #[inline] + fn ensure_buffered_at(&mut self, at: usize) -> Option { + if at >= self.len { + return None; + } + + let buf_end = self.buf_start + self.buf.len(); + if at >= self.buf_start && at < buf_end { + return Some(at - self.buf_start); + } + + // Refill aligned to chunk boundary to avoid cross-page decompression. + self.buf.clear(); + let aligned = (at / self.chunk_size) * self.chunk_size; + let end = (aligned + self.chunk_size).min(self.len); + self.buf_start = aligned; + self.source.read_into_at(aligned, end, &mut self.buf); + + if self.buf.is_empty() { + None + } else { + Some(at - aligned) + } + } +} diff --git a/crates/vecdb/src/error/mod.rs b/crates/vecdb/src/error/mod.rs new file mode 100644 index 000000000..999fa60e2 --- /dev/null +++ b/crates/vecdb/src/error/mod.rs @@ -0,0 +1,127 @@ +use std::{fmt, fs, io, result, time}; + +use thiserror::Error; + +#[cfg(feature = "zerocopy")] +mod zerocopy; + +use crate::{Format, Stamp, Version}; + +pub type Result = result::Result; + +/// Error types for vecdb operations. +#[derive(Debug, Error)] +pub enum Error { + #[error(transparent)] + IO(#[from] io::Error), + #[error(transparent)] + Format(#[from] fmt::Error), + #[error("Couldn't lock file. It must be already opened by another process.")] + TryLockError(#[from] fs::TryLockError), + #[cfg(feature = "zerocopy")] + #[error("ZeroCopy error")] + ZeroCopyError, + #[error(transparent)] + SystemTimeError(#[from] time::SystemTimeError), + #[cfg(feature = "pco")] + #[error(transparent)] + PCO(#[from] pco::errors::PcoError), + #[cfg(feature = "lz4")] + #[error(transparent)] + LZ4(#[from] lz4_flex::block::DecompressError), + #[error(transparent)] + RawDB(#[from] rawdb::Error), + #[cfg(feature = "serde_json")] + #[error(transparent)] + SerdeJSON(#[from] serde_json::Error), + #[cfg(feature = "sonic-rs")] + #[error(transparent)] + SonicRS(#[from] sonic_rs::Error), + + #[error("Wrong length: received: {received:?}, expected: {expected:?}")] + WrongLength { received: usize, expected: usize }, + #[error("Wrong endian")] + WrongEndian, + #[error("Iterator ended")] + IteratorEnded, + #[error("Different version received: {received:?}, expected: {expected:?}")] + DifferentVersion { + received: Version, + expected: Version, + }, + #[error("Index too high: index: {index}, len: {len}, name: {name}")] + IndexTooHigh { + index: usize, + len: usize, + name: String, + }, + #[error("Unexpected index: expected {expected}, got {got} ({name})")] + UnexpectedIndex { + expected: usize, + got: usize, + name: String, + }, + #[error("Expect vec to have index")] + ExpectVecToHaveIndex, + #[error("Failed to convert key to usize")] + FailedKeyTryIntoUsize, + #[error("Different format received: {received:?}, expected: {expected:?}")] + DifferentFormat { received: Format, expected: Format }, + #[error("Version cannot be zero, can't verify endianness otherwise")] + VersionCannotBeZero, + #[error("Stamp mismatch: file stamp {file:?} != vec stamp {vec:?}")] + StampMismatch { file: Stamp, vec: Stamp }, + #[error("Corrupted region: invalid length {region_len}")] + CorruptedRegion { name: String, region_len: usize }, + #[error("Decompression mismatch: expected {expected_len} values, got {actual_len}")] + DecompressionMismatch { + expected_len: usize, + actual_len: usize, + }, + #[error("Cannot remove PcodecVec: pages still referenced")] + PagesStillReferenced, + #[error("Invalid format byte: {0}")] + InvalidFormat(u8), + #[error("Invalid argument: {0}")] + InvalidArgument(&'static str), + #[error("Arithmetic overflow")] + Overflow, + #[error("Arithmetic underflow")] + Underflow, +} + +impl Error { + /// Returns true if this error is due to a file lock (another process has the database open). + /// Lock errors are transient and should not trigger data deletion. + pub fn is_lock_error(&self) -> bool { + matches!(self, Error::TryLockError(_)) + } + + /// Returns true if this error indicates data corruption or version incompatibility. + /// These errors may require resetting/deleting the data to recover. + pub fn is_data_error(&self) -> bool { + match self { + Error::IO(io_err) => is_io_data_error(io_err), + Error::RawDB(rawdb::Error::IO(io_err)) => is_io_data_error(io_err), + Error::RawDB(rawdb::Error::CorruptedMetadata(_)) => true, + Error::RawDB(rawdb::Error::InvalidMetadataSize { .. }) => true, + Error::RawDB(rawdb::Error::EmptyMetadata) => true, + Error::DifferentVersion { .. } + | Error::DifferentFormat { .. } + | Error::StampMismatch { .. } + | Error::CorruptedRegion { .. } + | Error::DecompressionMismatch { .. } + | Error::WrongEndian + | Error::WrongLength { .. } + | Error::InvalidFormat(_) => true, + _ => false, + } + } +} + +fn is_io_data_error(io_err: &io::Error) -> bool { + matches!( + io_err.kind(), + io::ErrorKind::IsADirectory | io::ErrorKind::NotADirectory + ) +} diff --git a/crates/vecdb/src/error/zerocopy.rs b/crates/vecdb/src/error/zerocopy.rs new file mode 100644 index 000000000..e8682f2e6 --- /dev/null +++ b/crates/vecdb/src/error/zerocopy.rs @@ -0,0 +1,13 @@ +use super::Error; + +impl From> for Error { + fn from(_: zerocopy::error::ConvertError) -> Self { + Self::ZeroCopyError + } +} + +impl From> for Error { + fn from(_: zerocopy::error::SizeError) -> Self { + Self::ZeroCopyError + } +} diff --git a/crates/vecdb/src/exit/guard.rs b/crates/vecdb/src/exit/guard.rs new file mode 100644 index 000000000..9401a837c --- /dev/null +++ b/crates/vecdb/src/exit/guard.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; + +use parking_lot::RwLock; + +/// Owned read guard for [`super::Exit`]. Can be moved across threads. +/// +/// parking_lot's `RawRwLock` supports cross-thread unlock, +/// so sending this guard to another thread is safe. +pub struct ExitGuard(Arc>); + +impl ExitGuard { + pub(super) fn new(lock: &Arc>) -> Self { + let arc = Arc::clone(lock); + use parking_lot::lock_api::RawRwLock as _; + // SAFETY:we release the lock in Drop. + unsafe { arc.raw().lock_shared() }; + Self(arc) + } +} + +impl Drop for ExitGuard { + fn drop(&mut self) { + use parking_lot::lock_api::RawRwLock as _; + // SAFETY:we acquired the shared lock in `new`, so we must release it. + unsafe { self.0.raw().unlock_shared() }; + } +} + +// SAFETY: parking_lot's RawRwLock supports unlock from a different thread than lock. +unsafe impl Send for ExitGuard {} +unsafe impl Sync for ExitGuard {} diff --git a/crates/vecdb/src/exit/mod.rs b/crates/vecdb/src/exit/mod.rs new file mode 100644 index 000000000..914728a93 --- /dev/null +++ b/crates/vecdb/src/exit/mod.rs @@ -0,0 +1,113 @@ +use std::{ + process::exit, + sync::{ + Arc, + atomic::{AtomicBool, AtomicI32, Ordering}, + }, + thread, +}; + +use log::info; +use parking_lot::{Mutex, RwLock}; + +mod guard; + +pub use guard::ExitGuard; + +static SIGNAL_RECEIVED: AtomicBool = AtomicBool::new(false); +static SIGNAL_PIPE: AtomicI32 = AtomicI32::new(-1); + +extern "C" fn signal_handler(_sig: libc::c_int) { + if SIGNAL_RECEIVED.swap(true, Ordering::Relaxed) { + const MSG: &[u8] = b"Shutdown already pending...\n"; + unsafe { libc::write(2, MSG.as_ptr().cast(), MSG.len()) }; + } else { + const MSG: &[u8] = b"Signal received, shutdown pending...\n"; + unsafe { libc::write(2, MSG.as_ptr().cast(), MSG.len()) }; + let fd = SIGNAL_PIPE.load(Ordering::Relaxed); + unsafe { libc::write(fd, b"x".as_ptr().cast(), 1) }; + } +} + +type Callbacks = Arc>>>; + +/// Graceful shutdown coordinator for ensuring data consistency during program exit. +/// +/// On first signal, a background thread acquires the write lock (waiting only for the +/// current critical section to finish), runs cleanup callbacks, and exits. +/// On second signal, force-exits immediately via `_exit(1)`. +#[derive(Default, Clone)] +pub struct Exit { + lock: Arc>, + cleanup_callbacks: Callbacks, +} + +impl Exit { + pub fn new() -> Self { + Self { + lock: Arc::new(RwLock::new(())), + cleanup_callbacks: Arc::new(Mutex::new(Vec::new())), + } + } + + /// Registers a callback to be executed during shutdown. + /// Callbacks are executed in registration order before the program exits. + pub fn register_cleanup(&self, callback: F) + where + F: Fn() + Send + Sync + 'static, + { + self.cleanup_callbacks.lock().push(Box::new(callback)); + } + + /// Registers signal handlers and spawns a background shutdown thread. + /// + /// # Panics + /// Panics if pipe creation or `sigaction` fails. + pub fn set_ctrlc_handler(&self) { + let mut fds = [0i32; 2]; + assert!( + unsafe { libc::pipe(fds.as_mut_ptr()) } == 0, + "failed to create pipe" + ); + + let read_fd = fds[0]; + SIGNAL_PIPE.store(fds[1], Ordering::Relaxed); + + unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = signal_handler as *const () as usize; + libc::sigemptyset(&raw mut action.sa_mask); + action.sa_flags = libc::SA_RESTART; + + assert!( + libc::sigaction(libc::SIGINT, &action, std::ptr::null_mut()) == 0, + "failed to install SIGINT handler" + ); + assert!( + libc::sigaction(libc::SIGTERM, &action, std::ptr::null_mut()) == 0, + "failed to install SIGTERM handler" + ); + } + + let lock = self.lock.clone(); + let callbacks = self.cleanup_callbacks.clone(); + thread::spawn(move || { + let mut buf = [0u8; 1]; + unsafe { libc::read(read_fd, buf.as_mut_ptr().cast(), 1) }; + + let _guard = lock.write(); + for callback in callbacks.lock().iter() { + callback(); + } + info!("Exiting..."); + exit(0); + }); + } + + /// Acquires a read lock to protect a critical section from shutdown. + /// The shutdown thread will wait for all read locks to be released before exiting. + /// Returns an owned guard that is Send + 'static (can be moved to background threads). + pub fn lock(&self) -> ExitGuard { + ExitGuard::new(&self.lock) + } +} diff --git a/crates/vecdb/src/iterators/mod.rs b/crates/vecdb/src/iterators/mod.rs new file mode 100644 index 000000000..548cb397d --- /dev/null +++ b/crates/vecdb/src/iterators/mod.rs @@ -0,0 +1,3 @@ +mod writer; + +pub use writer::*; diff --git a/crates/vecdb/src/iterators/writer.rs b/crates/vecdb/src/iterators/writer.rs new file mode 100644 index 000000000..916504c6f --- /dev/null +++ b/crates/vecdb/src/iterators/writer.rs @@ -0,0 +1,32 @@ +use crate::{Error, Formattable, Result, VecValue}; + +/// Stateful writer for streaming values one at a time to a string buffer. +/// +/// Useful for incremental serialization when memory constraints prevent +/// materializing entire collections. +pub trait ValueWriter { + /// Writes the next value to the buffer in CSV format. + /// + /// # Errors + /// Returns `Error::WrongLength` when no more values are available. + fn write_next(&mut self, buf: &mut String) -> Result<()>; +} + +/// Iterator-backed writer that formats values as CSV. +pub struct VecIteratorWriter { + pub iter: std::vec::IntoIter, +} + +impl ValueWriter for VecIteratorWriter +where + T: VecValue + Formattable, +{ + fn write_next(&mut self, buf: &mut String) -> Result<()> { + if let Some(value) = self.iter.next() { + value.fmt_csv(buf)?; + Ok(()) + } else { + Err(Error::IteratorEnded) + } + } +} diff --git a/crates/vecdb/src/lib.rs b/crates/vecdb/src/lib.rs new file mode 100644 index 000000000..84bcf8572 --- /dev/null +++ b/crates/vecdb/src/lib.rs @@ -0,0 +1,46 @@ +#![doc = include_str!("../README.md")] + +pub use rawdb::{Database, Error as RawDBError, PAGE_SIZE, Reader, likely, unlikely}; + +#[cfg(feature = "derive")] +pub use vecdb_derive::{Bytes, Pco}; + +mod base; +mod bytes; +mod cursor; +mod error; +mod exit; +mod iterators; +mod ops; +mod stamp; +mod traits; +mod variants; +mod version; + +use variants::*; + +pub use base::*; +pub use bytes::*; +pub use cursor::*; +pub use error::*; +pub use exit::*; +pub use iterators::*; +pub use ops::*; +pub use stamp::*; +pub use traits::*; +pub use variants::*; +pub use version::*; + +const ONE_KIB: usize = 1024; + +/// Buffer size for reading compressed data (512 KiB). +/// Chosen to balance memory usage with I/O efficiency - large enough to +/// amortize syscall overhead while fitting comfortably in L2/L3 cache. +const BUFFER_SIZE: usize = 512 * ONE_KIB; + +const SIZE_OF_U64: usize = std::mem::size_of::(); + +/// Crossover threshold in bytes for choosing IO vs mmap iteration strategy. +/// Ranges smaller than this use mmap (zero-copy), larger use buffered IO. +/// IO is kept for truly massive datasets that may exceed available address space. +pub(crate) const MMAP_CROSSOVER_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB diff --git a/crates/vecdb/src/ops/checked_sub.rs b/crates/vecdb/src/ops/checked_sub.rs new file mode 100644 index 000000000..25b16e202 --- /dev/null +++ b/crates/vecdb/src/ops/checked_sub.rs @@ -0,0 +1,19 @@ +/// Subtraction that returns `None` on underflow. +/// +/// Blanket-implemented for all primitive integer types. +pub trait CheckedSub: Sized { + fn checked_sub(self, rhs: Rhs) -> Option; +} + +macro_rules! impl_checked_sub { + ($($t:ty)*) => ($( + impl CheckedSub for $t { + #[inline] + fn checked_sub(self, rhs: Self) -> Option { + <$t>::checked_sub(self, rhs) + } + } + )*) +} + +impl_checked_sub! { i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize } diff --git a/crates/vecdb/src/ops/mod.rs b/crates/vecdb/src/ops/mod.rs new file mode 100644 index 000000000..664c30ca0 --- /dev/null +++ b/crates/vecdb/src/ops/mod.rs @@ -0,0 +1,7 @@ +//! Numeric traits (overflow-safe arithmetic) shared across the crate. + +mod checked_sub; +mod saturating_add; + +pub use checked_sub::*; +pub use saturating_add::*; diff --git a/crates/vecdb/src/ops/saturating_add.rs b/crates/vecdb/src/ops/saturating_add.rs new file mode 100644 index 000000000..2e15cab69 --- /dev/null +++ b/crates/vecdb/src/ops/saturating_add.rs @@ -0,0 +1,19 @@ +/// Addition that clamps to the type's min/max on overflow instead of wrapping. +/// +/// Blanket-implemented for all primitive integer types. +pub trait SaturatingAdd: Sized { + fn saturating_add(self, rhs: Rhs) -> Self; +} + +macro_rules! impl_saturating_add { + ($($t:ty)*) => ($( + impl SaturatingAdd for $t { + #[inline] + fn saturating_add(self, rhs: Self) -> Self { + <$t>::saturating_add(self, rhs) + } + } + )*) +} + +impl_saturating_add! { i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize } diff --git a/crates/vecdb/src/stamp/bytes.rs b/crates/vecdb/src/stamp/bytes.rs new file mode 100644 index 000000000..dcbb5fe82 --- /dev/null +++ b/crates/vecdb/src/stamp/bytes.rs @@ -0,0 +1,17 @@ +use crate::{Bytes, Result}; + +use super::Stamp; + +impl Bytes for Stamp { + type Array = [u8; size_of::()]; + + #[inline] + fn to_bytes(&self) -> Self::Array { + self.0.to_bytes() + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> Result { + Ok(Self(u64::from_bytes(bytes)?)) + } +} diff --git a/crates/vecdb/src/stamp/conversions.rs b/crates/vecdb/src/stamp/conversions.rs new file mode 100644 index 000000000..6e7548525 --- /dev/null +++ b/crates/vecdb/src/stamp/conversions.rs @@ -0,0 +1,13 @@ +use super::Stamp; + +impl From for Stamp { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl From for u64 { + fn from(value: Stamp) -> Self { + value.0 + } +} diff --git a/crates/vecdb/src/stamp/mod.rs b/crates/vecdb/src/stamp/mod.rs new file mode 100644 index 000000000..a09fbaacc --- /dev/null +++ b/crates/vecdb/src/stamp/mod.rs @@ -0,0 +1,15 @@ +mod bytes; +mod conversions; + +/// Marker for tracking when data was last modified. +/// +/// Used for change tracking, rollback support, and ETag generation. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[must_use = "Stamp values should be used for tracking"] +pub struct Stamp(pub(super) u64); + +impl Stamp { + pub fn new(stamp: u64) -> Self { + Self(stamp) + } +} diff --git a/crates/vecdb/src/traits/any.rs b/crates/vecdb/src/traits/any.rs new file mode 100644 index 000000000..ee2d2a653 --- /dev/null +++ b/crates/vecdb/src/traits/any.rs @@ -0,0 +1,59 @@ +use crate::{Stamp, Version, vec_region_name}; + +/// Converts an i64 index to usize, supporting negative indexing. +/// Negative indices count from the end. +pub fn i64_to_usize(i: i64, len: usize) -> usize { + if i >= 0 { + (i as usize).min(len) + } else { + let v = len as i64 + i; + if v < 0 { 0 } else { v as usize } + } +} + +/// Common trait for all vectors providing metadata and utility methods. +pub trait AnyVec: Send + Sync { + fn version(&self) -> Version; + fn name(&self) -> &str; + fn len(&self) -> usize; + #[inline] + fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Returns the string representation of the index type. + fn index_type_to_string(&self) -> &'static str; + /// Returns the combined name of the vector. + #[inline] + fn region_name(&self) -> String { + vec_region_name(self.name(), self.index_type_to_string()) + } + /// Returns the list of region names used by this vector. + fn region_names(&self) -> Vec; + /// Returns the size in bytes of the value type. + fn value_type_to_size_of(&self) -> usize; + /// Returns the short type name of the value type (e.g., "Sats", "StoredF64"). + fn value_type_to_string(&self) -> &'static str; + /// Generates an ETag for this vector based on stamp and optional end index. + fn etag(&self, stamp: Stamp, to: Option) -> String { + let len = self.len(); + format!( + "{}-{}-{}", + to.map_or(len, |to| { + if to.is_negative() { + len.saturating_sub(to.unsigned_abs() as usize) + } else { + to as usize + } + }), + usize::from(self.version()), + u64::from(stamp), + ) + } + + /// Converts an i64 index to usize, supporting negative indexing (Python-style). + #[inline] + fn i64_to_usize(&self, i: i64) -> usize { + let len = self.len(); + i64_to_usize(i, len) + } +} diff --git a/crates/vecdb/src/traits/any_exportable.rs b/crates/vecdb/src/traits/any_exportable.rs new file mode 100644 index 000000000..9ceab9b7d --- /dev/null +++ b/crates/vecdb/src/traits/any_exportable.rs @@ -0,0 +1,9 @@ +use super::{AnySerializableVec, AnyVecWithWriter}; + +/// Type-erased trait for vectors that are both writable and serializable. +/// This trait is automatically implemented for any type that implements both +/// `AnyVecWithWriter` and `AnySerializableVec`. +pub trait AnyExportableVec: AnyVecWithWriter + AnySerializableVec {} + +/// Blanket implementation for all types that implement both traits. +impl AnyExportableVec for V where V: AnyVecWithWriter + AnySerializableVec {} diff --git a/crates/vecdb/src/traits/any_readable.rs b/crates/vecdb/src/traits/any_readable.rs new file mode 100644 index 000000000..e8b344e85 --- /dev/null +++ b/crates/vecdb/src/traits/any_readable.rs @@ -0,0 +1,24 @@ +use crate::{AnyVec, ReadableVec, TypedVec, i64_to_usize}; + +/// Type-erased trait for collectable vectors. +pub trait AnyReadableVec: AnyVec { + /// Returns the number of items in the specified range. + fn range_count(&self, from: Option, to: Option) -> usize { + let len = self.len(); + let from = from.map(|i| i64_to_usize(i, len)).unwrap_or_default(); + let to = to.map(|i| i64_to_usize(i, len)).unwrap_or(len); + to.saturating_sub(from) + } + + /// Returns the total size in bytes of items in the specified range. + fn range_weight(&self, from: Option, to: Option) -> usize { + self.range_count(from, to) * self.value_type_to_size_of() + } +} + +impl AnyReadableVec for V +where + V: TypedVec, + V: ReadableVec, +{ +} diff --git a/crates/vecdb/src/traits/any_serializable.rs b/crates/vecdb/src/traits/any_serializable.rs new file mode 100644 index 000000000..a4c13e719 --- /dev/null +++ b/crates/vecdb/src/traits/any_serializable.rs @@ -0,0 +1,107 @@ +#[cfg(feature = "serde")] +use crate::Formattable; + +use super::AnyReadableVec; + +/// Type-erased trait for serializable vectors. +pub trait AnySerializableVec: AnyReadableVec { + /// Write JSON array to output buffer + #[cfg(feature = "serde")] + fn write_json( + &self, + from: Option, + to: Option, + buf: &mut Vec, + ) -> crate::Result<()>; + + /// Write single JSON value to output buffer (first value in range) + #[cfg(feature = "serde")] + fn write_json_value(&self, from: Option, buf: &mut Vec) -> crate::Result<()>; + + /// Return the last value as a serde_json::Value, or None if empty + #[cfg(feature = "serde_json")] + fn last_json_value(&self) -> Option; + + /// Write all values as CSV cells (newline-separated) directly without materializing a Vec. + fn write_csv_column( + &self, + from: Option, + to: Option, + buf: &mut String, + ) -> crate::Result<()>; +} + +#[cfg(feature = "serde")] +impl AnySerializableVec for V +where + V: crate::TypedVec, + V: crate::ReadableVec, + V::T: serde::Serialize + crate::Formattable, +{ + fn write_json( + &self, + from: Option, + to: Option, + buf: &mut Vec, + ) -> crate::Result<()> { + let len = self.len(); + let from_idx = from.unwrap_or(0); + let to_idx = to.unwrap_or(len).min(len); + + let count = to_idx.saturating_sub(from_idx); + buf.reserve(count * 20 + 2); + + buf.push(b'['); + let mut first = true; + self.for_each_range_at(from_idx, to_idx, |value: V::T| { + if !first { + buf.push(b','); + } + first = false; + value.fmt_json(buf); + }); + buf.push(b']'); + + Ok(()) + } + + fn write_json_value(&self, from: Option, buf: &mut Vec) -> crate::Result<()> { + let idx = from.unwrap_or(0); + if let Some(value) = self.collect_one_at(idx) { + value.fmt_json(buf); + } + + Ok(()) + } + + #[cfg(feature = "serde_json")] + fn last_json_value(&self) -> Option { + let len = self.len(); + if len == 0 { + return None; + } + let value: V::T = self.collect_one_at(len - 1)?; + serde_json::to_value(&value).ok() + } + + fn write_csv_column( + &self, + from: Option, + to: Option, + buf: &mut String, + ) -> crate::Result<()> { + let len = self.len(); + let from_idx = from.unwrap_or(0); + let to_idx = to.unwrap_or(len).min(len); + + let count = to_idx.saturating_sub(from_idx); + buf.reserve(count * 20); + + self.for_each_range_at(from_idx, to_idx, |value: V::T| { + value.fmt_csv(buf).expect("csv formatting failed"); + buf.push('\n'); + }); + + Ok(()) + } +} diff --git a/crates/vecdb/src/traits/any_stored.rs b/crates/vecdb/src/traits/any_stored.rs new file mode 100644 index 000000000..cc66822f6 --- /dev/null +++ b/crates/vecdb/src/traits/any_stored.rs @@ -0,0 +1,87 @@ +use std::path::PathBuf; + +use rawdb::{Database, Region}; + +use crate::{AnyVec, Header, Result, Stamp}; + +/// Trait for stored vectors that persist data to disk (as opposed to lazy computed vectors). +pub trait AnyStoredVec: AnyVec { + fn db_path(&self) -> PathBuf; + + fn region(&self) -> &Region; + + fn header(&self) -> &Header; + + fn mut_header(&mut self) -> &mut Header; + + /// Number of stamped change files to keep for rollback support. + fn saved_stamped_changes(&self) -> u16; + + /// Writes pending changes to storage. + /// Returns `Ok(true)` if data was written, `Ok(false)` if nothing to write. + #[doc(hidden)] + fn write(&mut self) -> Result; + + #[doc(hidden)] + fn db(&self) -> Database; + + #[inline] + fn flush(&mut self) -> Result<()> { + if self.write()? { + self.region().flush()?; + } + Ok(()) + } + + /// The actual length stored on disk. + fn real_stored_len(&self) -> usize; + /// The effective stored length (may differ from real_stored_len during truncation). + fn stored_len(&self) -> usize; + + #[inline] + fn update_stamp(&mut self, stamp: Stamp) { + self.mut_header().update_stamp(stamp); + } + + #[inline] + fn stamp(&self) -> Stamp { + self.header().stamp() + } + + #[inline] + fn stamped_write(&mut self, stamp: Stamp) -> Result<()> { + self.update_stamp(stamp); + self.write()?; + Ok(()) + } + + /// Flushes with the given stamp, saving changes to enable rollback. + /// Prefixed with `any_` to avoid conflict with `WritableVec::stamped_write_with_changes`. + fn any_stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()>; + + /// Flushes with the given stamp, optionally saving changes for rollback. + #[inline] + fn any_stamped_write_maybe_with_changes( + &mut self, + stamp: Stamp, + with_changes: bool, + ) -> Result<()> { + if with_changes { + self.any_stamped_write_with_changes(stamp) + } else { + self.stamped_write(stamp) + } + } + + fn serialize_changes(&self) -> Result>; + + /// Removes this vector's region from the database. + fn remove(self) -> Result<()>; + + /// Truncates the vector to the given length if it is longer. + /// Prefixed with `any_` to avoid conflict with `WritableVec::truncate_if_needed_at`. + fn any_truncate_if_needed_at(&mut self, index: usize) -> Result<()>; + + /// Resets the vector state, clearing all data. + fn any_reset(&mut self) -> Result<()>; +} diff --git a/crates/vecdb/src/traits/any_with_schema.rs b/crates/vecdb/src/traits/any_with_schema.rs new file mode 100644 index 000000000..0bc30c820 --- /dev/null +++ b/crates/vecdb/src/traits/any_with_schema.rs @@ -0,0 +1,20 @@ +use schemars::{JsonSchema, Schema, SchemaGenerator}; + +use crate::{AnyVec, TypedVec}; + +/// Trait for vectors whose value type implements JsonSchema. +/// Provides access to the JSON Schema of the value type. +pub trait AnyVecWithSchema: AnyVec { + /// Returns the JSON Schema for the value type. + fn value_schema(&self) -> Schema; +} + +impl AnyVecWithSchema for V +where + V: TypedVec, + V::T: JsonSchema, +{ + fn value_schema(&self) -> Schema { + V::T::json_schema(&mut SchemaGenerator::default()) + } +} diff --git a/crates/vecdb/src/traits/any_with_writer.rs b/crates/vecdb/src/traits/any_with_writer.rs new file mode 100644 index 000000000..c6233d3f6 --- /dev/null +++ b/crates/vecdb/src/traits/any_with_writer.rs @@ -0,0 +1,26 @@ +use crate::{AnyReadableVec, Formattable, ReadableVec, TypedVec, ValueWriter, VecIteratorWriter}; + +/// Type-erased trait for vecs that can produce a boxed row-by-row [`ValueWriter`]. +pub trait AnyVecWithWriter: AnyReadableVec { + /// Create a value writer that can be advanced row by row + fn create_writer(&self, from: Option, to: Option) -> Box; +} + +impl AnyVecWithWriter for V +where + V: TypedVec, + V: ReadableVec, + V::T: Formattable, +{ + fn create_writer(&self, from: Option, to: Option) -> Box { + let from_usize = from.map(|i| self.i64_to_usize(i)).unwrap_or(0); + let to_usize = to + .map(|i| self.i64_to_usize(i)) + .unwrap_or_else(|| self.len()); + + let values = self.collect_range_at(from_usize, to_usize); + Box::new(VecIteratorWriter { + iter: values.into_iter(), + }) + } +} diff --git a/crates/vecdb/src/traits/formattable.rs b/crates/vecdb/src/traits/formattable.rs new file mode 100644 index 000000000..84be8befb --- /dev/null +++ b/crates/vecdb/src/traits/formattable.rs @@ -0,0 +1,96 @@ +use std::fmt; + +/// Fast formatting trait that writes UTF-8 bytes directly into a buffer, +/// avoiding the `std::fmt` machinery for number types. +pub trait Formattable { + /// Write formatted UTF-8 bytes. Primary method — all others derive from it. + fn write_to(&self, buf: &mut Vec); + + /// Write to a String via write_to. + #[inline(always)] + fn fmt_into(&self, f: &mut String) { + // SAFETY: write_to produces valid UTF-8 (itoa/ryu/Display guarantee this). + unsafe { + self.write_to(f.as_mut_vec()); + } + } + + /// Write in CSV format. Override for types needing CSV escaping (e.g., quoting commas). + #[inline(always)] + fn fmt_csv(&self, f: &mut String) -> fmt::Result { + self.fmt_into(f); + Ok(()) + } + + /// Write in JSON format. Override for types needing JSON wrapping (e.g., string quotes). + #[inline(always)] + fn fmt_json(&self, buf: &mut Vec) { + self.write_to(buf); + } +} + +macro_rules! impl_formattable_int { + ($($t:ty),*) => { + $( + impl Formattable for $t { + #[inline(always)] + fn write_to(&self, buf: &mut Vec) { + let mut b = itoa::Buffer::new(); + buf.extend_from_slice(b.format(*self).as_bytes()); + } + } + )* + }; +} + +impl_formattable_int!( + u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize +); + +macro_rules! impl_formattable_float { + ($($t:ty),*) => { + $( + impl Formattable for $t { + #[inline(always)] + fn write_to(&self, buf: &mut Vec) { + let mut b = ryu::Buffer::new(); + buf.extend_from_slice(b.format(*self).as_bytes()); + } + } + )* + }; +} + +impl_formattable_float!(f32, f64); + +impl Formattable for bool { + #[inline(always)] + fn write_to(&self, buf: &mut Vec) { + buf.extend_from_slice(if *self { b"true" } else { b"false" }); + } +} + +impl Formattable for Option { + #[inline] + fn write_to(&self, buf: &mut Vec) { + if let Some(v) = self { + v.write_to(buf); + } + } + + #[inline] + fn fmt_csv(&self, f: &mut String) -> fmt::Result { + if let Some(v) = self { + v.fmt_csv(f)?; + } + Ok(()) + } + + #[inline] + fn fmt_json(&self, buf: &mut Vec) { + match self { + Some(v) => v.fmt_json(buf), + None => buf.extend_from_slice(b"null"), + } + } +} diff --git a/crates/vecdb/src/traits/importable.rs b/crates/vecdb/src/traits/importable.rs new file mode 100644 index 000000000..3c4418b2d --- /dev/null +++ b/crates/vecdb/src/traits/importable.rs @@ -0,0 +1,21 @@ +use rawdb::Database; + +use crate::{ImportOptions, Result, Version}; + +/// Trait for types that can be imported from a database. +/// +/// This provides a uniform interface for constructing stored vectors, +/// enabling generic wrappers like `EagerVec` to work with any storage format. +pub trait ImportableVec: Sized { + /// Import from database, creating if needed. + fn import(db: &Database, name: &str, version: Version) -> Result; + + /// Import with custom options. + fn import_with(options: ImportOptions) -> Result; + + /// Import from database, resetting on version/format mismatch. + fn forced_import(db: &Database, name: &str, version: Version) -> Result; + + /// Forced import with custom options. + fn forced_import_with(options: ImportOptions) -> Result; +} diff --git a/crates/vecdb/src/traits/index.rs b/crates/vecdb/src/traits/index.rs new file mode 100644 index 000000000..03a84ba7d --- /dev/null +++ b/crates/vecdb/src/traits/index.rs @@ -0,0 +1,57 @@ +use std::{fmt::Debug, ops::Add}; + +use crate::PrintableIndex; + +/// Trait for types that can be used as vector indices. +/// +/// This trait is automatically implemented for any type that satisfies the +/// required bounds. No manual implementation is needed. +pub trait VecIndex +where + Self: Debug + + Default + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + From + + Into + + Add + + Send + + Sync + + PrintableIndex + + 'static, +{ + /// Converts this index to a `usize`. + #[inline] + fn to_usize(self) -> usize { + self.into() + } + + /// Returns the previous index, or `None` if this is zero. + #[inline] + fn decremented(self) -> Option { + self.to_usize().checked_sub(1).map(Self::from) + } +} + +impl VecIndex for I where + I: Debug + + Default + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + From + + Into + + Add + + Send + + Sync + + PrintableIndex + + 'static +{ +} diff --git a/crates/vecdb/src/traits/mod.rs b/crates/vecdb/src/traits/mod.rs new file mode 100644 index 000000000..111b495e6 --- /dev/null +++ b/crates/vecdb/src/traits/mod.rs @@ -0,0 +1,43 @@ +mod any; +mod any_exportable; +mod any_readable; +mod any_serializable; +mod any_stored; +#[cfg(feature = "schemars")] +mod any_with_schema; +mod any_with_writer; +mod formattable; +mod importable; +mod index; +mod printable; +mod read_only_clone; +mod readable; +mod readable_option; +mod storage_mode; +mod stored; +mod typed; +mod value; +mod value_strategy; +pub(crate) mod writable; + +pub use any::*; +pub use any_exportable::*; +pub use any_readable::*; +pub use any_serializable::*; +pub use any_stored::*; +#[cfg(feature = "schemars")] +pub use any_with_schema::*; +pub use any_with_writer::*; +pub use formattable::*; +pub use importable::*; +pub use index::*; +pub use printable::*; +pub use read_only_clone::*; +pub use readable::*; +pub use readable_option::*; +pub use storage_mode::*; +pub use stored::*; +pub use typed::*; +pub use value::*; +pub use value_strategy::*; +pub use writable::*; diff --git a/crates/vecdb/src/traits/printable.rs b/crates/vecdb/src/traits/printable.rs new file mode 100644 index 000000000..7f2a949e9 --- /dev/null +++ b/crates/vecdb/src/traits/printable.rs @@ -0,0 +1,99 @@ +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +/// Extracts the short type name from a full type path and caches it. +pub fn short_type_name() -> &'static str { + static CACHE: OnceLock>> = OnceLock::new(); + + let full: &'static str = std::any::type_name::(); + + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + let mut guard = cache.lock().unwrap(); + + if let Some(&short) = guard.get(full) { + return short; + } + + let short_owned = shorten_type_name(full); + + let short: &'static str = Box::leak(short_owned.into_boxed_str()); + guard.insert(full, short); + short +} + +/// Shortens a type name by removing path prefixes, including inside generics. +/// `some::module::Close` -> `Close` +/// +/// Also unwraps `Option` to just `T` since Option is a serialization +/// concern (null in JSON) not a type identity. +fn shorten_type_name(full: &str) -> String { + let generic_start = full.find('<'); + + let (base, generics) = match generic_start { + Some(pos) => (&full[..pos], Some(&full[pos + 1..full.len() - 1])), + None => (full, None), + }; + + let short_base = base.rsplit("::").next().unwrap_or(base); + + // Option is a serialization concern (null in JSON), not a type identity. + if short_base == "Option" + && let Some(inner) = generics + { + return shorten_type_name(inner.trim()); + } + + match generics { + Some(params) => { + let shortened_params = split_generic_params(params) + .into_iter() + .map(|p| shorten_type_name(p.trim())) + .collect::>() + .join(", "); + format!("{}<{}>", short_base, shortened_params) + } + None => short_base.to_string(), + } +} + +/// Split generic parameters respecting nested angle brackets. +/// `A, B, E` -> ["A", "B", "E"] +fn split_generic_params(params: &str) -> Vec<&str> { + let mut result = Vec::new(); + let mut depth = 0; + let mut start = 0; + + for (i, c) in params.char_indices() { + match c { + '<' => depth += 1, + '>' => depth -= 1, + ',' if depth == 0 => { + result.push(¶ms[start..i]); + start = i + 1; + } + _ => {} + } + } + result.push(¶ms[start..]); + result +} + +/// Provides string representations of index types for display and region naming. +pub trait PrintableIndex { + /// Returns the canonical string name for this index type. + fn to_string() -> &'static str; + + /// Returns all accepted string representations for this index type. + /// Used for parsing and type identification. + fn to_possible_strings() -> &'static [&'static str]; +} + +impl PrintableIndex for usize { + fn to_string() -> &'static str { + "usize" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["usize"] + } +} diff --git a/crates/vecdb/src/traits/read_only_clone.rs b/crates/vecdb/src/traits/read_only_clone.rs new file mode 100644 index 000000000..fb00dad57 --- /dev/null +++ b/crates/vecdb/src/traits/read_only_clone.rs @@ -0,0 +1,50 @@ +use std::collections::BTreeMap; + +use crate::StoredVec; + +/// Trait for creating read-only clones of composite types. +/// +/// For stored vecs, delegates to [`StoredVec::read_only_clone`]. +/// For containers (Option, BTreeMap, cohort groups), propagates to inner types. +/// For Traversable-derived types with `M: StorageMode`, the derive macro +/// generates an impl that maps `Self` → `Self`. +pub trait ReadOnlyClone { + type ReadOnly; + fn read_only_clone(&self) -> Self::ReadOnly; +} + +impl ReadOnlyClone for V { + type ReadOnly = V::ReadOnly; + + #[inline] + fn read_only_clone(&self) -> V::ReadOnly { + ::read_only_clone(self) + } +} + +impl ReadOnlyClone for Option { + type ReadOnly = Option; + + #[inline] + fn read_only_clone(&self) -> Option { + self.as_ref().map(ReadOnlyClone::read_only_clone) + } +} + +impl ReadOnlyClone for [T; N] { + type ReadOnly = [T::ReadOnly; N]; + + fn read_only_clone(&self) -> [T::ReadOnly; N] { + self.each_ref().map(ReadOnlyClone::read_only_clone) + } +} + +impl ReadOnlyClone for BTreeMap { + type ReadOnly = BTreeMap; + + fn read_only_clone(&self) -> BTreeMap { + self.iter() + .map(|(k, v)| (k.clone(), v.read_only_clone())) + .collect() + } +} diff --git a/crates/vecdb/src/traits/readable.rs b/crates/vecdb/src/traits/readable.rs new file mode 100644 index 000000000..865ab51ed --- /dev/null +++ b/crates/vecdb/src/traits/readable.rs @@ -0,0 +1,490 @@ +use std::ops::AddAssign; + +use crate::{AnyVec, VecIndex, VecValue, cursor::Cursor}; + +/// Default chunk size for chunked iteration (matches PcoVec page size). +pub const READ_CHUNK_SIZE: usize = 4096; + +/// High-performance reading of vector values. +/// +/// This is the primary trait for reading data from any vec type — stored, compressed, +/// lazy, or computed. All methods see the full state including uncommitted (pushed) values. +/// +/// # Method overview +/// +/// | Method | Use when | +/// |--------|----------| +/// | `for_each` / `for_each_range` | Processing every element, static dispatch | +/// | `fold` / `fold_range` | Accumulating a result (SIMD-optimized on stored vecs) | +/// | `collect` / `collect_range` | Materializing values into a `Vec` | +/// | `collect_one` / `collect_first` / `collect_last` | Materializing a single value | +/// | `for_each_range_dyn` | Trait-object contexts (`&dyn ReadableVec`) | +/// | `try_fold_range` | Fold with early exit on error | +/// | `read_into` / `cursor` | Sequential access with buffer reuse | +/// +/// # Typed vs `_at` methods +/// +/// Methods like `collect_one(I)`, `fold_range(I, I, …)` accept the typed index `I`. +/// The `_at` variants (`collect_one_at(usize)`, `fold_range_at(usize, usize, …)`) +/// accept raw `usize` and are what implementations override. +/// +/// # Point reads +/// +/// For raw vecs, use `VecReader::get()` for O(1) random access. +/// For sequential persisted raw reads, use `vec.reader().cursor()` to avoid +/// the general cursor's staging buffer. +/// For any vec through the trait, use `collect_one(i)` — this materializes +/// a single value (decodes a page for compressed vecs). +/// +/// # Performance +/// +/// Stored vecs override `fold_range_at` and `try_fold_range_at` to delegate to their +/// internal source's optimized `fold()`, enabling SIMD auto-vectorization. +/// +/// The default `fold_range_at` uses chunked `read_into_at` calls with a monomorphized +/// inner loop — LLVM can auto-vectorize this without `&mut dyn FnMut` overhead. +/// +/// For maximum throughput on stored vecs, prefer `fold_range` / `for_each_range` +/// with static dispatch (`&impl ReadableVec` or concrete type). +pub trait ReadableVec: AnyVec { + // ── Required ───────────────────────────────────────────────────── + + /// Preferred number of values per cursor refill. + /// + /// Compressed vectors return their format page size so sequential cursors + /// decode each page once. Other vectors use [`READ_CHUNK_SIZE`]. + #[inline] + fn cursor_chunk_size(&self) -> usize { + READ_CHUNK_SIZE + } + + /// Appends elements in `[from, to)` to `buf`. + /// + /// Implementations MUST NOT clear `buf` — they only append. This enables + /// chunked fold (clear + fill per chunk) and multi-range reads (append + /// multiple ranges into one buffer). + /// + /// Object-safe: `&mut Vec` is a concrete type, no `Self: Sized` needed. + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec); + + /// Iterates over `[from, to)` by raw index, calling `f` for each value. + /// + /// Object-safe: callable on `&dyn ReadableVec`. Every implementor must + /// provide this — there is intentionally no default to prevent silent + /// fallback to a slow buffered path. + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)); + + /// Folds over `[from, to)` by raw index with an accumulator. + /// + /// Every implementor must provide an optimal path — there is intentionally + /// no default to prevent silent fallback to a slow buffered path. + fn fold_range_at B>(&self, from: usize, to: usize, init: B, f: F) -> B + where + Self: Sized; + + /// Fallible fold over `[from, to)` by raw index with early exit on error. + /// + /// Every implementor must provide an optimal path — there is intentionally + /// no default to prevent silent fallback to a slow buffered path. + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized; + + // ── Typed-index wrappers ─────────────────────────────────────────── + + /// Appends elements in `[from, to)` to `buf` using typed indices. + #[inline] + fn read_into(&self, from: I, to: I, buf: &mut Vec) { + self.read_into_at(from.to_usize(), to.to_usize(), buf) + } + + /// Creates a forward-only `Cursor` that reuses an internal buffer across + /// chunked `read_into_at` calls. One allocation for the lifetime of the cursor. + #[inline] + fn cursor(&self) -> Cursor<'_, I, T, Self> + where + Self: Sized, + { + Cursor::new(self) + } + + /// Iterates over `[from, to)` by typed index, calling `f` for each value (object-safe). + #[inline] + fn for_each_range_dyn(&self, from: I, to: I, f: &mut dyn FnMut(T)) { + self.for_each_range_dyn_at(from.to_usize(), to.to_usize(), f) + } + + /// Folds over `[from, to)` by typed index with an accumulator. + #[inline] + fn fold_range B>(&self, from: I, to: I, init: B, f: F) -> B + where + Self: Sized, + { + self.fold_range_at(from.to_usize(), to.to_usize(), init, f) + } + + /// Fallible fold over `[from, to)` by typed index with early exit on error. + #[inline] + fn try_fold_range std::result::Result>( + &self, + from: I, + to: I, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized, + { + self.try_fold_range_at(from.to_usize(), to.to_usize(), init, f) + } + + /// Calls `f` for each value in `[from, to)` by typed index. Requires `Sized`. + #[inline] + fn for_each_range(&self, from: I, to: I, f: F) + where + Self: Sized, + { + self.for_each_range_at(from.to_usize(), to.to_usize(), f) + } + + /// Collects values in `[from, to)` by typed index into a `Vec`. + #[inline] + fn collect_range(&self, from: I, to: I) -> Vec + where + Self: Sized, + { + self.collect_range_at(from.to_usize(), to.to_usize()) + } + + /// Collects a single value at typed `index`, or `None` if out of bounds. + #[inline] + fn collect_one(&self, index: I) -> Option { + self.collect_one_at(index.to_usize()) + } + + /// Returns the minimum value in `[from, to)` by typed index, or `None` if empty. + #[inline] + fn min(&self, from: I, to: I) -> Option + where + Self: Sized, + T: PartialOrd, + { + self.min_at(from.to_usize(), to.to_usize()) + } + + /// Returns the maximum value in `[from, to)` by typed index, or `None` if empty. + #[inline] + fn max(&self, from: I, to: I) -> Option + where + Self: Sized, + T: PartialOrd, + { + self.max_at(from.to_usize(), to.to_usize()) + } + + /// Returns the sum of values in `[from, to)` by typed index, or `None` if empty. + #[inline] + fn sum(&self, from: I, to: I) -> Option + where + Self: Sized, + T: AddAssign + From, + { + self.sum_at(from.to_usize(), to.to_usize()) + } + + // ── Raw-index convenience (all have defaults) ────────────────────── + + /// Calls `f` for each value in `[from, to)` by raw index. Requires `Sized` (static dispatch). + #[inline] + fn for_each_range_at(&self, from: usize, to: usize, mut f: F) + where + Self: Sized, + { + self.fold_range_at(from, to, (), |(), v| f(v)); + } + + /// Fallible for-each over `[from, to)` by raw index with early exit on error. + #[inline] + fn try_for_each_range_at std::result::Result<(), E>>( + &self, + from: usize, + to: usize, + mut f: F, + ) -> std::result::Result<(), E> + where + Self: Sized, + { + self.try_fold_range_at(from, to, (), |(), v| f(v)) + } + + /// Calls `f` for every value in the vector. + #[inline] + fn for_each(&self, f: F) + where + Self: Sized, + { + self.for_each_range_at(0, self.len(), f); + } + + /// Folds over all values with an accumulator. + #[inline] + fn fold B>(&self, init: B, f: F) -> B + where + Self: Sized, + { + self.fold_range_at(0, self.len(), init, f) + } + + /// Collects values in `[from, to)` by raw index into a `Vec`. + #[inline] + fn collect_range_at(&self, from: usize, to: usize) -> Vec + where + Self: Sized, + { + self.collect_range_dyn(from, to) + } + + /// Clears `buf` then fills it with values in `[from, to)`. Reuses the buffer allocation. + #[inline] + fn collect_range_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + buf.clear(); + self.read_into_at(from, to, buf); + } + + /// Collects values in `[from, to)` into a `Vec` (object-safe). + #[inline] + fn collect_range_dyn(&self, from: usize, to: usize) -> Vec { + let mut buf = Vec::with_capacity(to.saturating_sub(from)); + self.read_into_at(from, to, &mut buf); + buf + } + + /// Collects all values into a `Vec`. + #[inline] + fn collect(&self) -> Vec + where + Self: Sized, + { + self.collect_range_at(0, self.len()) + } + + /// Collects a single value at raw `index`, or `None` if out of bounds. + /// + /// Uses `for_each_range_dyn_at` with a stack-local `Option`. Stored vecs + /// override `for_each_range_dyn_at` → `fold_range_at` for zero-alloc reads; + /// lazy vecs override `collect_one_at` directly. + #[inline(always)] + fn collect_one_at(&self, index: usize) -> Option { + if index >= self.len() { + return None; + } + let mut result = None; + self.for_each_range_dyn_at(index, index + 1, &mut |v| result = Some(v)); + result + } + + /// Collects the first value, or `None` if empty. + #[inline] + fn collect_first(&self) -> Option { + self.collect_one_at(0) + } + + /// Collects the last value, or `None` if empty. + #[inline] + fn collect_last(&self) -> Option { + let len = self.len(); + if len > 0 { + self.collect_one_at(len - 1) + } else { + None + } + } + + /// Collects values using signed indices. Negative indices count from the end + /// (Python-style): `-1` is the last element, `-2` is second-to-last, etc. + #[inline] + fn collect_signed_range(&self, from: Option, to: Option) -> Vec + where + Self: Sized, + { + let from = from.map(|i| self.i64_to_usize(i)).unwrap_or(0); + let to = to + .map(|i| self.i64_to_usize(i)) + .unwrap_or_else(|| self.len()); + self.collect_range_at(from, to) + } + + /// Collects values using signed indices (object-safe). + #[inline] + fn collect_signed_range_dyn(&self, from: Option, to: Option) -> Vec { + let from = from.map(|i| self.i64_to_usize(i)).unwrap_or(0); + let to = to + .map(|i| self.i64_to_usize(i)) + .unwrap_or_else(|| self.len()); + self.collect_range_dyn(from, to) + } + + /// Collects all values into a `Vec` (object-safe). + #[inline] + fn collect_dyn(&self) -> Vec { + self.collect_range_dyn(0, self.len()) + } + + // ── Sparse reads ────────────────────────────────────────────── + + /// Reads values at specific sorted indices (ascending), skipping holes. + /// + /// Default uses a forward-only [`Cursor`] — each underlying page is read + /// at most once. Lazy vecs override to map indices to their source and + /// call this method recursively, bottoming out at stored vecs. + /// + /// `indices` **must** be sorted ascending. Output order matches `indices`. + fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { + let mut cursor = Cursor::new(self); + out.reserve(indices.len()); + indices.iter().for_each(|&i| { + if let Some(v) = cursor.get(i) { + out.push(v); + } + }); + } + + /// Reads values at specific sorted indices (ascending), returning a new `Vec`. + #[inline] + fn read_sorted_at(&self, indices: &[usize]) -> Vec { + let mut out = Vec::with_capacity(indices.len()); + self.read_sorted_into_at(indices, &mut out); + out + } + + /// Reads values at specific sorted typed indices, appending to `out`. + #[inline] + fn read_sorted_into(&self, indices: &[I], out: &mut Vec) { + let raw: Vec = indices.iter().map(|i| i.to_usize()).collect(); + self.read_sorted_into_at(&raw, out); + } + + /// Reads values at specific sorted typed indices, returning a new `Vec`. + #[inline] + fn read_sorted(&self, indices: &[I]) -> Vec { + let mut out = Vec::with_capacity(indices.len()); + self.read_sorted_into(indices, &mut out); + out + } + + // ── Aggregations ─────────────────────────────────────────────── + + /// Returns the minimum value in `[from, to)` by raw index, or `None` if empty. + #[inline] + fn min_at(&self, from: usize, to: usize) -> Option + where + Self: Sized, + T: PartialOrd, + { + self.fold_range_at(from, to, None, |acc: Option, v| match acc { + Some(cur) if cur <= v => Some(cur), + _ => Some(v), + }) + } + + /// Returns the minimum value in `[from, to)`, or `None` if empty (object-safe). + #[inline] + fn min_dyn(&self, from: usize, to: usize) -> Option + where + T: PartialOrd, + { + let mut result: Option = None; + self.for_each_range_dyn_at(from, to, &mut |v| match &result { + Some(cur) if *cur <= v => {} + _ => result = Some(v), + }); + result + } + + /// Returns the maximum value in `[from, to)` by raw index, or `None` if empty. + #[inline] + fn max_at(&self, from: usize, to: usize) -> Option + where + Self: Sized, + T: PartialOrd, + { + self.fold_range_at(from, to, None, |acc: Option, v| match acc { + Some(cur) if cur >= v => Some(cur), + _ => Some(v), + }) + } + + /// Returns the maximum value in `[from, to)`, or `None` if empty (object-safe). + #[inline] + fn max_dyn(&self, from: usize, to: usize) -> Option + where + T: PartialOrd, + { + let mut result: Option = None; + self.for_each_range_dyn_at(from, to, &mut |v| match &result { + Some(cur) if *cur >= v => {} + _ => result = Some(v), + }); + result + } + + /// Returns the sum of values in `[from, to)` by raw index, or `None` if empty. + #[inline] + fn sum_at(&self, from: usize, to: usize) -> Option + where + Self: Sized, + T: AddAssign + From, + { + let mut has_values = false; + let result = self.fold_range_at(from, to, T::from(0), |mut acc, v| { + acc += v; + has_values = true; + acc + }); + has_values.then_some(result) + } + + /// Returns the sum of values in `[from, to)`, or `None` if empty (object-safe). + #[inline] + fn sum_dyn(&self, from: usize, to: usize) -> Option + where + T: AddAssign + From, + { + let mut result = T::from(0); + let mut has_values = false; + self.for_each_range_dyn_at(from, to, &mut |v| { + result += v; + has_values = true; + }); + has_values.then_some(result) + } +} + +/// Trait for readable vectors that can be cloned as trait objects. +pub trait ReadableCloneableVec: ReadableVec { + fn read_only_boxed_clone(&self) -> Box>; +} + +impl ReadableCloneableVec for U +where + U: 'static + ReadableVec + Clone, +{ + fn read_only_boxed_clone(&self) -> Box> { + Box::new(self.clone()) + } +} + +impl Clone for Box> { + fn clone(&self) -> Self { + self.read_only_boxed_clone() + } +} + +/// Type alias for boxed read-only vectors. +pub type ReadableBoxedVec = Box>; diff --git a/crates/vecdb/src/traits/readable_option.rs b/crates/vecdb/src/traits/readable_option.rs new file mode 100644 index 000000000..423fe20b5 --- /dev/null +++ b/crates/vecdb/src/traits/readable_option.rs @@ -0,0 +1,28 @@ +use crate::{ReadableVec, VecIndex, VecValue}; + +/// Extension methods for `ReadableVec>`. +pub trait ReadableOptionVec { + /// Collect all values, replacing `None` with `T::default()`. + fn collect_or_default(&self) -> Vec; + + /// Flattens `Option>` → `Option`. + fn collect_one_flat(&self, index: I) -> Option; +} + +impl ReadableOptionVec for V +where + V: ReadableVec> + Sized, + I: VecIndex, + T: VecValue + Default, +{ + fn collect_or_default(&self) -> Vec { + self.fold(Vec::with_capacity(self.len()), |mut v, opt| { + v.push(opt.unwrap_or_default()); + v + }) + } + + fn collect_one_flat(&self, index: I) -> Option { + self.collect_one(index).flatten() + } +} diff --git a/crates/vecdb/src/traits/storage_mode.rs b/crates/vecdb/src/traits/storage_mode.rs new file mode 100644 index 000000000..5a50710b9 --- /dev/null +++ b/crates/vecdb/src/traits/storage_mode.rs @@ -0,0 +1,26 @@ +use crate::{ReadableVec, StoredVec, TypedVec}; + +/// Marker trait that selects between read-write and read-only storage. +/// +/// Composite types use `M::Stored` for stored vec fields. +/// When `M = Rw`, the field is `V` itself (identity) with full write access. +/// When `M = Ro`, the field is `V::ReadOnly` — a lean read-only clone (~40-48 bytes). +pub trait StorageMode: 'static { + type Stored: TypedVec + + ReadableVec + + 'static; +} + +/// Read-write mode. `Stored` is the identity — the full read-write vec. +pub struct Rw; + +impl StorageMode for Rw { + type Stored = V; +} + +/// Read-only mode. `Stored` is `V::ReadOnly` — a lean clone for disk reads. +pub struct Ro; + +impl StorageMode for Ro { + type Stored = V::ReadOnly; +} diff --git a/crates/vecdb/src/traits/stored.rs b/crates/vecdb/src/traits/stored.rs new file mode 100644 index 000000000..fac7fbc01 --- /dev/null +++ b/crates/vecdb/src/traits/stored.rs @@ -0,0 +1,20 @@ +use crate::{ + ImportableVec, ReadableCloneableVec, ReadableVec, TypedVec, VecIndex, VecValue, WritableVec, +}; + +/// Super trait combining all common stored vec traits. +pub trait StoredVec: + ImportableVec + TypedVec + WritableVec + ReadableCloneableVec +where + Self::I: VecIndex, + Self::T: VecValue, +{ + /// The concrete lean read-only type returned by [`read_only_clone`](StoredVec::read_only_clone). + type ReadOnly: TypedVec + + ReadableVec + + Clone + + 'static; + + /// Creates a lean read-only clone that only carries fields needed for disk reads. + fn read_only_clone(&self) -> Self::ReadOnly; +} diff --git a/crates/vecdb/src/traits/typed.rs b/crates/vecdb/src/traits/typed.rs new file mode 100644 index 000000000..1308a7554 --- /dev/null +++ b/crates/vecdb/src/traits/typed.rs @@ -0,0 +1,16 @@ +use crate::{AnyVec, VecIndex, VecValue}; + +/// A vector with statically-known index and value types. +/// +/// This trait extends [`AnyVec`] by providing associated types for the index (`I`) +/// and value (`T`) types, enabling type-safe operations at compile time. +/// +/// # Type Parameters +/// - `I`: The index type, must implement [`VecIndex`] +/// - `T`: The value type, must implement [`VecValue`] +pub trait TypedVec: AnyVec { + /// The index type used to address elements in this vector. + type I: VecIndex; + /// The value type stored in this vector. + type T: VecValue; +} diff --git a/crates/vecdb/src/traits/value.rs b/crates/vecdb/src/traits/value.rs new file mode 100644 index 000000000..d561b99f4 --- /dev/null +++ b/crates/vecdb/src/traits/value.rs @@ -0,0 +1,13 @@ +use std::fmt::Debug; + +/// Marker trait for types that can be stored as values in a vector. +/// +/// This trait is automatically implemented for any type that satisfies the +/// required bounds. No manual implementation is needed. +pub trait VecValue +where + Self: Sized + Debug + Clone + Send + Sync + 'static, +{ +} + +impl VecValue for T where T: Sized + Debug + Clone + Send + Sync + 'static {} diff --git a/crates/vecdb/src/traits/value_strategy.rs b/crates/vecdb/src/traits/value_strategy.rs new file mode 100644 index 000000000..c5a6a45f1 --- /dev/null +++ b/crates/vecdb/src/traits/value_strategy.rs @@ -0,0 +1,48 @@ +use crate::Result; + +/// Value serialization strategy shared by all vec types (raw and compressed). +/// +/// Handles reading/writing individual values to/from bytes. +pub trait ValueStrategy: Send + Sync + Clone { + /// Whether T has native memory layout (can be memcpy'd to/from bytes). + const IS_NATIVE_LAYOUT: bool = false; + + /// Deserializes a value from its byte representation. + fn read(bytes: &[u8]) -> Result; + + /// Serializes a value by appending its byte representation to the buffer. + fn write_to_vec(value: &T, buf: &mut Vec); + + /// Serializes a value directly into a fixed-size slice. + fn write_to_slice(value: &T, dst: &mut [u8]); +} + +/// Implements `ValueStrategy` for a Bytes-based strategy type. +/// Used by all Bytes-based strategies (BytesStrategy, LZ4Strategy, ZstdStrategy, PcodecStrategy). +macro_rules! impl_bytes_value_strategy { + ($strategy:ident, $value_trait:path) => { + impl $crate::ValueStrategy for $strategy + where + T: $value_trait, + { + const IS_NATIVE_LAYOUT: bool = T::IS_NATIVE_LAYOUT; + + #[inline(always)] + fn read(bytes: &[u8]) -> $crate::Result { + T::from_bytes(bytes) + } + + #[inline(always)] + fn write_to_vec(value: &T, buf: &mut Vec) { + buf.extend_from_slice(value.to_bytes().as_ref()); + } + + #[inline(always)] + fn write_to_slice(value: &T, dst: &mut [u8]) { + dst.copy_from_slice(value.to_bytes().as_ref()); + } + } + }; +} + +pub(crate) use impl_bytes_value_strategy; diff --git a/crates/vecdb/src/traits/writable.rs b/crates/vecdb/src/traits/writable.rs new file mode 100644 index 000000000..83c831640 --- /dev/null +++ b/crates/vecdb/src/traits/writable.rs @@ -0,0 +1,222 @@ +use std::{collections::BTreeMap, path::PathBuf}; + +use log::info; +use rawdb::unlikely; + +use crate::{AnyStoredVec, Error, Result, Stamp, VecIndex, VecValue, Version}; + +/// Maximum in-memory cache size before forcing a flush (1 GiB). +/// Prevents unbounded memory growth when pushing many values without flushing. +pub(crate) const MAX_CACHE_SIZE: usize = 1024 * 1024 * 1024; + +/// Typed interface for stored vectors (push, truncate, rollback). +/// +/// Provides the core write operations for all stored vec types. +/// For reading, use [`ReadableVec`] (`collect_range`, `fold_range`, etc.). +/// Raw vecs (`BytesVec`, `ZeroCopyVec`) additionally provide +/// `VecReader` for O(1) random access. +/// +/// [`ReadableVec`]: crate::ReadableVec +pub trait WritableVec: AnyStoredVec +where + I: VecIndex, + T: VecValue, +{ + const SIZE_OF_T: usize = size_of::(); + + fn push(&mut self, value: T); + + /// Returns the current pushed (uncommitted) values. + fn pushed(&self) -> &[T]; + + /// Truncates the vector to the given usize index if the current length exceeds it. + fn truncate_if_needed_at(&mut self, index: usize) -> Result<()>; + + /// Resets the vector state. + fn reset(&mut self) -> Result<()>; + + /// Resets uncommitted changes. + fn reset_unsaved(&mut self); + + /// Returns true if there are uncommitted changes. + fn is_dirty(&self) -> bool; + + /// Flushes with the given stamp, saving changes to enable rollback. + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()>; + + /// Rolls back the most recent change set. + fn rollback(&mut self) -> Result<()>; + + /// Returns available rollback change files. + #[doc(hidden)] + fn find_rollback_files(&self) -> Result>; + + /// Saves type-specific rollback state after the rollback loop completes. + #[doc(hidden)] + fn save_rollback_state(&mut self); + + /// Rolls back changes to before the given stamp. + fn rollback_before(&mut self, stamp: Stamp) -> Result { + let files = self.find_rollback_files()?; + + // Walk change files newest-first. Each rollback decrements the vec + // stamp to the previous file's stamp; if they ever disagree, the + // change-file chain is broken (missing or corrupt). + for (&file_stamp, _) in files.range(..=self.stamp()).rev() { + let current = self.stamp(); + if current < stamp { + break; + } + if file_stamp != current { + return Err(Error::StampMismatch { + file: file_stamp, + vec: current, + }); + } + self.rollback()?; + } + + self.save_rollback_state(); + Ok(self.stamp()) + } + + /// Number of pushed (uncommitted) values in the memory buffer. + #[inline] + fn pushed_len(&self) -> usize { + self.pushed().len() + } + + /// Returns true if there are no pushed (uncommitted) values. + #[inline] + fn is_pushed_empty(&self) -> bool { + self.pushed_len() == 0 + } + + /// Returns true if the typed index is within bounds. + #[inline] + fn has(&self, index: I) -> bool { + self.has_at(index.to_usize()) + } + + /// Returns true if the usize index is within bounds. + #[inline] + fn has_at(&self, index: usize) -> bool { + index < self.len() + } + + /// Pushes a value at the given index, erroring if index != current length. + /// Use this when you expect to always append in order. + #[inline] + fn checked_push(&mut self, index: I, value: T) -> Result<()> { + self.checked_push_at(index.to_usize(), value) + } + + /// Pushes a value at the given usize index, erroring if index != current length. + /// Use this when you expect to always append in order. + #[inline] + fn checked_push_at(&mut self, index: usize, value: T) -> Result<()> { + let len = self.len(); + + if unlikely(index != len) { + return Err(Error::UnexpectedIndex { + expected: len, + got: index, + name: self.name().to_string(), + }); + } + + self.push(value); + Ok(()) + } + + /// Truncates the vector to the given index if the current length exceeds it. + fn truncate_if_needed(&mut self, index: I) -> Result<()> { + self.truncate_if_needed_at(index.to_usize()) + } + + /// Truncates the vector to the given index if needed, updating the stamp. + #[inline] + fn truncate_if_needed_with_stamp(&mut self, index: I, stamp: Stamp) -> Result<()> { + self.update_stamp(stamp); + self.truncate_if_needed(index) + } + + /// Clears all values from the vector. + #[inline] + fn clear(&mut self) -> Result<()> { + self.truncate_if_needed_at(0) + } + + /// Returns true if the pushed cache has reached the batch limit (~1GiB). + /// + /// When this limit is reached, the caller should flush to disk before continuing. + /// This prevents excessive memory usage during bulk operations. + #[inline] + fn batch_limit_reached(&self) -> bool { + self.pushed_len() * Self::SIZE_OF_T >= MAX_CACHE_SIZE + } + + /// Extends the vector to `target_len`, filling with `value`. + /// Batches writes in ~1GB chunks to avoid memory explosion. + fn fill_to(&mut self, target_len: usize, value: T) -> Result<()> + where + T: Copy, + { + let batch_count = MAX_CACHE_SIZE / Self::SIZE_OF_T.max(1); + + while self.len() < target_len { + let count = (target_len - self.len()).min(batch_count); + for _ in 0..count { + self.push(value); + } + if self.batch_limit_reached() { + self.write()?; + } + } + Ok(()) + } + + /// Flushes with the given stamp, optionally saving changes for rollback. + #[inline] + fn stamped_write_maybe_with_changes(&mut self, stamp: Stamp, with_changes: bool) -> Result<()> { + if with_changes { + self.stamped_write_with_changes(stamp) + } else { + self.stamped_write(stamp) + } + } + + /// Validates the computed version against the stored version, resetting if they don't match. + /// Automatically includes the vec's own version - only pass dependency versions. + fn validate_computed_version_or_reset(&mut self, dep_version: Version) -> Result<()> { + let version = self.header().vec_version() + dep_version; + if version != self.header().computed_version() { + self.mut_header().update_computed_version(version); + if !self.is_empty() { + self.reset()?; + } + } + + if self.is_empty() { + info!( + "Computing {}_to_{}...", + self.index_type_to_string(), + self.name() + ) + } + + Ok(()) + } + + /// Validates computed version and truncates to `max_from` in one call. + /// + /// Equivalent to: + /// ```ignore + /// vec.validate_computed_version_or_reset(dep_version)?; + /// vec.truncate_if_needed(max_from)?; + /// ``` + fn validate_and_truncate(&mut self, dep_version: Version, max_from: I) -> Result<()> { + self.validate_computed_version_or_reset(dep_version)?; + self.truncate_if_needed(max_from) + } +} diff --git a/crates/vecdb/src/variants/cached/any_vec.rs b/crates/vecdb/src/variants/cached/any_vec.rs new file mode 100644 index 000000000..b7a8edd54 --- /dev/null +++ b/crates/vecdb/src/variants/cached/any_vec.rs @@ -0,0 +1,40 @@ +use crate::{AnyVec, TypedVec, Version, short_type_name}; + +use super::CachedVec; + +impl AnyVec for CachedVec { + #[inline(always)] + fn version(&self) -> Version { + self.inner.version() + } + + #[inline(always)] + fn name(&self) -> &str { + self.inner.name() + } + + #[inline(always)] + fn len(&self) -> usize { + self.inner.len() + } + + #[inline(always)] + fn index_type_to_string(&self) -> &'static str { + self.inner.index_type_to_string() + } + + #[inline(always)] + fn region_names(&self) -> Vec { + self.inner.region_names() + } + + #[inline(always)] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline(always)] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } +} diff --git a/crates/vecdb/src/variants/cached/budget.rs b/crates/vecdb/src/variants/cached/budget.rs new file mode 100644 index 000000000..4646b858c --- /dev/null +++ b/crates/vecdb/src/variants/cached/budget.rs @@ -0,0 +1,28 @@ +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +/// Budget gate for [`super::CachedVec`] materialization. +/// +/// When the budget is exhausted, reads fall through to the inner vec without caching. +pub trait CachedVecBudget: Send + Sync { + /// Attempts to reserve one cache slot given the entry's access count. + /// Implementations may enforce a minimum access threshold or evict entries. + fn try_reserve(&self, access_count: u64) -> bool; +} + +impl CachedVecBudget for AtomicUsize { + #[inline] + fn try_reserve(&self, _: u64) -> bool { + self.fetch_update(Relaxed, Relaxed, |n| if n > 0 { Some(n - 1) } else { None }) + .is_ok() + } +} + +/// Budget that always allows materialization (used by [`super::CachedVec::wrap`]). +pub struct NoBudget; + +impl CachedVecBudget for NoBudget { + #[inline] + fn try_reserve(&self, _: u64) -> bool { + true + } +} diff --git a/crates/vecdb/src/variants/cached/clone.rs b/crates/vecdb/src/variants/cached/clone.rs new file mode 100644 index 000000000..6e26f9c39 --- /dev/null +++ b/crates/vecdb/src/variants/cached/clone.rs @@ -0,0 +1,15 @@ +use crate::TypedVec; + +use super::CachedVec; + +impl Clone for CachedVec { + #[inline(always)] + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + cache: self.cache.clone(), + budget: self.budget, + access_count: self.access_count.clone(), + } + } +} diff --git a/crates/vecdb/src/variants/cached/cloneable.rs b/crates/vecdb/src/variants/cached/cloneable.rs new file mode 100644 index 000000000..b146c9e18 --- /dev/null +++ b/crates/vecdb/src/variants/cached/cloneable.rs @@ -0,0 +1,52 @@ +use std::sync::Arc; + +use crate::{AnyVec, ReadOnlyClone, ReadableVec, StoredVec, TypedVec, VecIndex, VecValue}; + +use super::CachedVec; + +pub trait CachedReadableVec: AnyVec + Send + Sync +where + I: VecIndex, + T: VecValue, +{ + fn cached(&self) -> Arc<[T]>; + fn cached_boxed_clone(&self) -> CachedBoxedVec; +} + +pub type CachedBoxedVec = Box>; + +impl Clone for CachedBoxedVec +where + I: VecIndex, + T: VecValue, +{ + fn clone(&self) -> Self { + self.cached_boxed_clone() + } +} + +impl CachedReadableVec for CachedVec +where + I: VecIndex, + T: VecValue, + V: TypedVec + ReadableVec + Clone + Send + Sync + 'static, +{ + fn cached(&self) -> Arc<[T]> { + self.cached() + } + + fn cached_boxed_clone(&self) -> CachedBoxedVec { + Box::new(self.clone()) + } +} + +impl CachedVec +where + V: StoredVec, + V::ReadOnly: + TypedVec + ReadableVec + Clone + Send + Sync + 'static, +{ + pub fn read_only_cached_boxed_clone(&self) -> CachedBoxedVec { + Box::new(self.read_only_clone()) + } +} diff --git a/crates/vecdb/src/variants/cached/mod.rs b/crates/vecdb/src/variants/cached/mod.rs new file mode 100644 index 000000000..a659ba0e1 --- /dev/null +++ b/crates/vecdb/src/variants/cached/mod.rs @@ -0,0 +1,142 @@ +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering::Relaxed}, +}; + +use parking_lot::RwLock; + +mod any_vec; +mod budget; +mod clone; +mod cloneable; +mod read_only_clone; +mod readable; +mod typed; + +pub use budget::{CachedVecBudget, NoBudget}; +pub use cloneable::{CachedBoxedVec, CachedReadableVec}; + +use crate::{ReadOnlyClone, ReadableVec, StoredVec, TypedVec, VecIndex, Version}; + +static NO_BUDGET: NoBudget = NoBudget; + +/// Cached wrapper around any readable vec, refreshed when len or version changes. +/// +/// Wraps a concrete vec `V` and adds an in-memory cache layer. +/// Reads check the cache first; on miss, the inner vec is read and cached. +/// +/// For writes, access the inner vec directly via the `inner` field. +/// +/// When constructed with a budget, materialization is gated: if the budget +/// is exhausted, reads fall through to the inner vec without caching. +pub struct CachedVec { + pub inner: V, + #[allow(clippy::type_complexity)] + pub(super) cache: Arc)>>, + pub(super) budget: &'static dyn CachedVecBudget, + pub(super) access_count: Option>, +} + +impl CachedVec { + fn empty() -> (usize, Version, Arc<[V::T]>) { + (0, Version::ZERO, Arc::from(&[] as &[V::T])) + } + + pub fn wrap(inner: V) -> Self { + Self { + inner, + cache: Arc::new(RwLock::new(Self::empty())), + budget: &NO_BUDGET, + access_count: None, + } + } + + pub fn wrap_budgeted( + inner: V, + budget: &'static dyn CachedVecBudget, + access_count: Arc, + ) -> Self { + Self { + inner, + cache: Arc::new(RwLock::new(Self::empty())), + budget, + access_count: Some(access_count), + } + } + + #[inline(always)] + pub fn version(&self) -> Version { + self.inner.version() + } + + pub fn clear(&self) { + *self.cache.write() = Self::empty(); + if let Some(c) = &self.access_count { + c.store(0, Relaxed); + } + } +} + +impl> CachedVec { + /// Returns the full cached snapshot. Always materializes on miss (ignores budget). + #[inline(always)] + pub fn cached(&self) -> Arc<[V::T]> { + self.materialize(false).unwrap() + } + + /// Returns the value at the given typed index from the cached snapshot. + #[inline(always)] + pub fn get(&self, index: V::I) -> Option { + self.get_at(index.to_usize()) + } + + /// Returns the value at the given raw index from the cached snapshot. + #[inline(always)] + pub fn get_at(&self, index: usize) -> Option { + self.cached().get(index).cloned() + } + + /// Returns `None` when budget is exhausted or below min access threshold. + #[inline] + pub(super) fn try_cached(&self) -> Option> { + self.materialize(true) + } + + fn materialize(&self, check_budget: bool) -> Option> { + let len = self.inner.len(); + let version = self.inner.version(); + + let count = self + .access_count + .as_ref() + .map(|c| c.fetch_add(1, Relaxed) + 1) + .unwrap_or(0); + + let cache = self.cache.read(); + if cache.0 == len && cache.1 == version { + return Some(cache.2.clone()); + } + drop(cache); + + if check_budget && !self.budget.try_reserve(count) { + return None; + } + + let data: Arc<[V::T]> = self.inner.collect_range_dyn(0, len).into(); + let mut cache = self.cache.write(); + if cache.0 == len && cache.1 == version { + return Some(cache.2.clone()); + } + *cache = (len, version, data.clone()); + + Some(data) + } +} + +impl CachedVec { + /// Boxes a read-only clone for use with type-erased APIs (e.g. LazyVecFrom1). + #[inline] + pub fn read_only_boxed_clone(&self) -> crate::ReadableBoxedVec { + Box::new(self.read_only_clone()) + } +} diff --git a/crates/vecdb/src/variants/cached/read_only_clone.rs b/crates/vecdb/src/variants/cached/read_only_clone.rs new file mode 100644 index 000000000..179211fe9 --- /dev/null +++ b/crates/vecdb/src/variants/cached/read_only_clone.rs @@ -0,0 +1,17 @@ +use crate::{ReadOnlyClone, StoredVec}; + +use super::CachedVec; + +impl ReadOnlyClone for CachedVec { + type ReadOnly = CachedVec; + + #[inline] + fn read_only_clone(&self) -> Self::ReadOnly { + CachedVec { + inner: self.inner.read_only_clone(), + cache: self.cache.clone(), + budget: self.budget, + access_count: self.access_count.clone(), + } + } +} diff --git a/crates/vecdb/src/variants/cached/readable.rs b/crates/vecdb/src/variants/cached/readable.rs new file mode 100644 index 000000000..eaef892f2 --- /dev/null +++ b/crates/vecdb/src/variants/cached/readable.rs @@ -0,0 +1,101 @@ +use crate::{ReadableVec, TypedVec}; + +use super::CachedVec; + +impl> ReadableVec for CachedVec { + #[inline] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + if let Some(data) = self.try_cached() { + let to = to.min(data.len()); + if from < to { + buf.extend_from_slice(&data[from..to]); + } + } else { + self.inner.read_into_at(from, to, buf); + } + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(V::T)) { + if let Some(data) = self.try_cached() { + let to = to.min(data.len()); + let from = from.min(to); + for v in &data[from..to] { + f(v.clone()); + } + } else { + self.inner.for_each_range_dyn_at(from, to, f); + } + } + + #[inline] + fn fold_range_at B>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> B + where + Self: Sized, + { + if let Some(data) = self.try_cached() { + let to = to.min(data.len()); + let from = from.min(to); + let mut acc = init; + for v in &data[from..to] { + acc = f(acc, v.clone()); + } + acc + } else { + self.inner.fold_range_at(from, to, init, f) + } + } + + #[inline] + fn try_fold_range_at Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> Result + where + Self: Sized, + { + if let Some(data) = self.try_cached() { + let to = to.min(data.len()); + let from = from.min(to); + let mut acc = init; + for v in &data[from..to] { + acc = f(acc, v.clone())?; + } + Ok(acc) + } else { + self.inner.try_fold_range_at(from, to, init, f) + } + } + + #[inline] + fn collect_one_at(&self, index: usize) -> Option { + if let Some(data) = self.try_cached() { + data.get(index).cloned() + } else { + self.inner.collect_one_at(index) + } + } + + #[inline] + fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { + if let Some(data) = self.try_cached() { + out.reserve(indices.len()); + for &i in indices { + if let Some(v) = data.get(i) { + out.push(v.clone()); + } + } + } else { + self.inner.read_sorted_into_at(indices, out); + } + } +} diff --git a/crates/vecdb/src/variants/cached/typed.rs b/crates/vecdb/src/variants/cached/typed.rs new file mode 100644 index 000000000..6c748f8d6 --- /dev/null +++ b/crates/vecdb/src/variants/cached/typed.rs @@ -0,0 +1,8 @@ +use crate::TypedVec; + +use super::CachedVec; + +impl TypedVec for CachedVec { + type I = V::I; + type T = V::T; +} diff --git a/crates/vecdb/src/variants/compressed/inner/mod.rs b/crates/vecdb/src/variants/compressed/inner/mod.rs new file mode 100644 index 000000000..abc179b8d --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/mod.rs @@ -0,0 +1,11 @@ +mod page; +mod pages; +mod read_only; +mod read_write; +mod strategy; + +pub use page::*; +pub use pages::*; +pub use read_only::*; +pub use read_write::*; +pub use strategy::*; diff --git a/crates/vecdb/src/variants/compressed/inner/page/bytes.rs b/crates/vecdb/src/variants/compressed/inner/page/bytes.rs new file mode 100644 index 000000000..03b80a8e1 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/page/bytes.rs @@ -0,0 +1,34 @@ +use crate::{Bytes, Error, Result}; + +use super::Page; + +impl Bytes for Page { + type Array = [u8; size_of::()]; + + fn to_bytes(&self) -> Self::Array { + let mut bytes = [0u8; 16]; + bytes[0..8].copy_from_slice(&self.start.to_bytes()); + bytes[8..12].copy_from_slice(&self.bytes.to_bytes()); + bytes[12..16].copy_from_slice(&self.values.to_bytes()); + bytes + } + + fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() < size_of::() { + return Err(Error::WrongLength { + expected: size_of::(), + received: bytes.len(), + }); + } + + let start = u64::from_bytes(&bytes[0..8])?; + let bytes_val = u32::from_bytes(&bytes[8..12])?; + let values = u32::from_bytes(&bytes[12..16])?; + + Ok(Self { + start, + bytes: bytes_val, + values, + }) + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/page/mod.rs b/crates/vecdb/src/variants/compressed/inner/page/mod.rs new file mode 100644 index 000000000..f6a7cadfa --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/page/mod.rs @@ -0,0 +1,54 @@ +mod bytes; + +/// Metadata for a page in a CompressedVec. +/// +/// Each page stores a chunk of values, either compressed or raw. +/// The high bit of `values` encodes whether the page is raw (uncompressed). +/// Raw pages are used for the last partial page to avoid recompression on every write. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct Page { + /// Absolute byte offset in the region where page data starts + pub start: u64, + /// Number of bytes on disk (compressed or raw) + pub bytes: u32, + /// Number of values in this page. High bit encodes raw flag. + pub(super) values: u32, +} + +impl Page { + const RAW_FLAG: u32 = 1 << 31; + + pub fn compressed(start: u64, bytes: u32, values: u32) -> Self { + debug_assert!(values & Self::RAW_FLAG == 0, "values too large"); + Self { + start, + bytes, + values, + } + } + + pub fn raw(start: u64, bytes: u32, values: u32) -> Self { + debug_assert!(values & Self::RAW_FLAG == 0, "values too large"); + Self { + start, + bytes, + values: values | Self::RAW_FLAG, + } + } + + #[inline] + pub fn is_raw(&self) -> bool { + self.values & Self::RAW_FLAG != 0 + } + + #[inline] + pub fn values_count(&self) -> u32 { + self.values & !Self::RAW_FLAG + } + + #[inline] + pub fn end(&self) -> u64 { + self.start + self.bytes as u64 + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/pages.rs b/crates/vecdb/src/variants/compressed/inner/pages.rs new file mode 100644 index 000000000..80696025c --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/pages.rs @@ -0,0 +1,121 @@ +use rawdb::{Database, Region, unlikely}; + +use crate::{Bytes, Error, HEADER_OFFSET, Result}; + +use super::Page; + +/// Manages page metadata for compressed vectors. +/// +/// Stores page metadata (offsets, sizes, value counts) separately from the +/// compressed data itself. This allows quick random access to pages without +/// scanning through compressed data. +/// +/// Uses incremental flushing to minimize disk writes - only writes changed pages. +#[derive(Debug, Clone)] +pub struct Pages { + region: Region, + vec: Vec, + /// Index of first changed page, or None if no changes + change_at: Option, +} + +impl Pages { + const SIZE_OF_PAGE: usize = size_of::(); + + pub fn import(db: &Database, name: &str) -> Result { + let region = db.create_region_if_needed(name)?; + + let vec = region + .create_reader() + .read_all() + .chunks(Self::SIZE_OF_PAGE) + .map(Page::from_bytes) + .collect::>()?; + + Ok(Self { + region, + vec, + change_at: None, + }) + } + + pub fn flush(&mut self) -> Result<()> { + let Some(change_at) = self.change_at.take() else { + return Ok(()); + }; + + let at = change_at * Self::SIZE_OF_PAGE; + let pages_to_write = self.vec.len() - change_at; + + let mut bytes = Vec::with_capacity(pages_to_write * Self::SIZE_OF_PAGE); + for page in &self.vec[change_at..] { + bytes.extend_from_slice(&page.to_bytes()); + } + + self.region.truncate_write(at, &bytes)?; + + Ok(()) + } + + pub fn len(&self) -> usize { + self.vec.len() + } + + pub fn get(&self, page_index: usize) -> Option<&Page> { + self.vec.get(page_index) + } + + pub fn last(&self) -> Option<&Page> { + self.vec.last() + } + + /// Pushes a new page, returning an error if the page_index doesn't match the current length. + pub fn checked_push(&mut self, page_index: usize, page: Page) -> Result<()> { + if unlikely(page_index != self.vec.len()) { + return Err(Error::UnexpectedIndex { + expected: self.vec.len(), + got: page_index, + name: self.region.meta().id().to_string(), + }); + } + + self.set_changed_at(page_index); + + self.vec.push(page); + Ok(()) + } + + fn set_changed_at(&mut self, page_index: usize) { + if self.change_at.is_none_or(|pi| pi > page_index) { + self.change_at.replace(page_index); + } + } + + pub fn reset(&mut self) { + self.truncate(0); + } + + pub fn truncate(&mut self, page_index: usize) -> Option { + let page = self.get(page_index).cloned(); + self.vec.truncate(page_index); + self.set_changed_at(page_index); + page + } + + pub fn next_start(&self) -> u64 { + self.last().map_or(HEADER_OFFSET as u64, |page| page.end()) + } + + pub fn stored_len(&self, per_page: usize) -> usize { + if let Some(last) = self.last() { + (self.len() - 1) * per_page + last.values_count() as usize + } else { + 0 + } + } + + pub fn remove(self) -> Result<()> { + self.region.remove()?; + Ok(()) + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_only/any_vec.rs b/crates/vecdb/src/variants/compressed/inner/read_only/any_vec.rs new file mode 100644 index 000000000..629159f4e --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_only/any_vec.rs @@ -0,0 +1,47 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name, vec_region_name}; + +use super::{super::CompressionStrategy, ReadOnlyCompressedVec}; + +impl AnyVec for ReadOnlyCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline] + fn version(&self) -> Version { + self.base.version() + } + + #[inline] + fn name(&self) -> &str { + self.base.name() + } + + #[inline] + fn len(&self) -> usize { + self.base.len() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + let base = vec_region_name(self.base.name(), I::to_string()); + let pages = format!("{base}_pages"); + vec![base, pages] + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_only/mod.rs b/crates/vecdb/src/variants/compressed/inner/read_only/mod.rs new file mode 100644 index 000000000..4eaa44afd --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_only/mod.rs @@ -0,0 +1,106 @@ +use std::{marker::PhantomData, sync::Arc}; + +use parking_lot::RwLock; + +mod any_vec; +mod readable; +mod typed; + +use crate::{ + CompressedIoSource, CompressedMmapSource, MMAP_CROSSOVER_BYTES, ReadOnlyBaseVec, VecIndex, + VecValue, +}; + +use super::{CompressionStrategy, Pages}; + +/// Lean read-only view of a compressed vector (~48 bytes). +/// +/// Carries only the fields needed for disk reads: region, shared length, +/// name/header metadata, and the pages index. No pushed buffer, no rollback state. +/// +/// Created via `ReadWriteCompressedVec::read_only_clone`. +#[derive(Debug, Clone)] +pub struct ReadOnlyCompressedVec { + pub(super) base: ReadOnlyBaseVec, + pub(super) pages: Arc>, + _strategy: PhantomData, +} + +impl ReadOnlyCompressedVec { + pub(crate) fn new(base: ReadOnlyBaseVec, pages: Arc>) -> Self { + Self { + base, + pages, + _strategy: PhantomData, + } + } +} + +impl ReadOnlyCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline(always)] + pub(super) fn fold_source B>( + &self, + from: usize, + to: usize, + len: usize, + init: B, + f: F, + ) -> B { + let range_bytes = (to - from) * size_of::(); + if range_bytes > MMAP_CROSSOVER_BYTES { + CompressedIoSource::::new_from_parts( + self.base.region(), + &self.pages, + len, + from, + to, + ) + .fold(init, f) + } else { + CompressedMmapSource::::new_from_parts( + self.base.region(), + &self.pages, + len, + from, + to, + ) + .fold(init, f) + } + } + + #[inline(always)] + pub(super) fn try_fold_source std::result::Result>( + &self, + from: usize, + to: usize, + len: usize, + init: B, + f: F, + ) -> std::result::Result { + let range_bytes = (to - from) * size_of::(); + if range_bytes > MMAP_CROSSOVER_BYTES { + CompressedIoSource::::new_from_parts( + self.base.region(), + &self.pages, + len, + from, + to, + ) + .try_fold(init, f) + } else { + CompressedMmapSource::::new_from_parts( + self.base.region(), + &self.pages, + len, + from, + to, + ) + .try_fold(init, f) + } + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_only/readable.rs b/crates/vecdb/src/variants/compressed/inner/read_only/readable.rs new file mode 100644 index 000000000..9646381dd --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_only/readable.rs @@ -0,0 +1,73 @@ +use crate::{READ_CHUNK_SIZE, ReadableVec, VecIndex, VecValue}; + +use super::{ + super::{CompressionStrategy, ReadWriteCompressedVec}, + ReadOnlyCompressedVec, +}; + +impl ReadableVec for ReadOnlyCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline(always)] + fn cursor_chunk_size(&self) -> usize { + let per_page = ReadWriteCompressedVec::::PER_PAGE; + per_page * READ_CHUNK_SIZE.div_ceil(per_page) + } + + #[inline(always)] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return; + } + buf.reserve(to - from); + + let reader = self.base.region().create_reader(); + let pages = self.pages.read(); + ReadWriteCompressedVec::::read_stored_pages_into(&reader, &pages, from, to, buf); + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + self.fold_range_at(from, to, (), |(), v| f(v)); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, f: F) -> B + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return init; + } + self.fold_source(from, to, len, init, f) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return Ok(init); + } + self.try_fold_source(from, to, len, init, f) + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_only/typed.rs b/crates/vecdb/src/variants/compressed/inner/read_only/typed.rs new file mode 100644 index 000000000..e385a5a92 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_only/typed.rs @@ -0,0 +1,13 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::{super::CompressionStrategy, ReadOnlyCompressedVec}; + +impl TypedVec for ReadOnlyCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/any_stored_vec.rs b/crates/vecdb/src/variants/compressed/inner/read_write/any_stored_vec.rs new file mode 100644 index 000000000..ffb5c76c3 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/any_stored_vec.rs @@ -0,0 +1,214 @@ +use std::{mem, path::PathBuf}; + +use rawdb::{Database, Region}; + +use crate::{AnyStoredVec, AnyVec, Error, Header, Result, Stamp, VecIndex, VecValue, WritableVec}; + +use super::super::{CompressionStrategy, Page}; +use super::ReadWriteCompressedVec; + +impl AnyStoredVec for ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline] + fn db_path(&self) -> PathBuf { + self.base.db_path() + } + + #[inline] + fn region(&self) -> &Region { + self.base.region() + } + + #[inline] + fn header(&self) -> &Header { + self.base.header() + } + + #[inline] + fn mut_header(&mut self) -> &mut Header { + self.base.mut_header() + } + + #[inline] + fn saved_stamped_changes(&self) -> u16 { + self.base.saved_stamped_changes() + } + + #[inline] + fn stored_len(&self) -> usize { + self.base.stored_len() + } + + #[inline] + fn real_stored_len(&self) -> usize { + self.pages.read().stored_len(Self::PER_PAGE) + } + + fn write(&mut self) -> Result { + self.base.write_header_if_needed()?; + + let stored_len = self.stored_len(); + let pushed_len = self.base.pushed().len(); + + let (truncate_at, starting_page_index, partial_page) = { + let pages = self.pages.read(); + + let real_stored_len = pages.stored_len(Self::PER_PAGE); + if stored_len > real_stored_len { + return Err(Error::CorruptedRegion { + name: self.name().to_string(), + region_len: real_stored_len, + }); + } + + if pushed_len == 0 && stored_len == real_stored_len { + return Ok(false); + } + + let starting_page_index = Self::index_to_page_index(stored_len); + if starting_page_index > pages.len() { + return Err(Error::CorruptedRegion { + name: self.name().to_string(), + region_len: pages.len(), + }); + } + + if starting_page_index < pages.len() { + let partial_len = stored_len % Self::PER_PAGE; + let page = *pages + .get(starting_page_index) + .ok_or(Error::ExpectVecToHaveIndex)?; + ( + page.start, + starting_page_index, + if partial_len != 0 { + Some((page, partial_len)) + } else { + None + }, + ) + } else { + (pages.next_start(), starting_page_index, None) + } + }; + // Pages lock released — decompression happens without blocking readers + + // Fast path: append to existing raw page without reading it back. + // When the last page is raw, not truncated, and won't overflow, just + // write the new pushed bytes at the end of the existing page data. + if let Some((page, partial_len)) = partial_page + && page.is_raw() + && partial_len == page.values_count() as usize + && partial_len + pushed_len < Self::PER_PAGE + { + let taken = mem::take(self.base.mut_pushed()); + let raw = S::values_to_bytes(&taken); + let append_at = page.end() as usize; + self.region().truncate_write(append_at, &raw)?; + + let mut pages = self.pages.write(); + pages.truncate(starting_page_index); + pages.checked_push( + starting_page_index, + Page::raw( + page.start, + page.bytes + raw.len() as u32, + (partial_len + pushed_len) as u32, + ), + )?; + self.base.update_stored_len(stored_len + pushed_len); + pages.flush()?; + return Ok(true); + } + + // Decompress the partial page outside the pages lock. + let mut values = if let Some((page, partial_len)) = partial_page { + let reader = self.create_reader(); + let data = reader.unchecked_read(page.start as usize, page.bytes as usize); + let mut page_values = S::decode_page(data, &page)?; + page_values.truncate(partial_len); + page_values + } else { + vec![] + }; + + // Encode pages with no locks held. Full pages compress; the last + // partial page is stored raw (avoids recompression on every write). + let taken = mem::take(self.base.mut_pushed()); + if values.is_empty() { + values = taken; + } else { + values.extend(taken); + } + + let num_pages = values.len().div_ceil(Self::PER_PAGE); + let mut buf = Vec::new(); + let mut page_sizes: Vec<(usize, usize, bool)> = Vec::with_capacity(num_pages); + for chunk in values.chunks(Self::PER_PAGE) { + let (encoded, is_raw) = if chunk.len() == Self::PER_PAGE { + (Self::compress_page(chunk)?, false) + } else { + (S::values_to_bytes(chunk), true) + }; + + if page_sizes.is_empty() { + // Size the aggregate from the first page's observed ratio + // instead of duplicating the entire uncompressed batch. + buf.reserve(encoded.len().saturating_mul(num_pages)); + } + page_sizes.push((encoded.len(), chunk.len(), is_raw)); + buf.extend_from_slice(&encoded); + } + + // Write the region before re-taking the pages lock to avoid deadlock. + self.region().truncate_write(truncate_at as usize, &buf)?; + + let mut pages = self.pages.write(); + pages.truncate(starting_page_index); + + for (i, &(byte_len, values_len, is_raw)) in page_sizes.iter().enumerate() { + let start = pages.next_start(); + let page = if is_raw { + Page::raw(start, byte_len as u32, values_len as u32) + } else { + Page::compressed(start, byte_len as u32, values_len as u32) + }; + pages.checked_push(starting_page_index + i, page)?; + } + + self.base.update_stored_len(stored_len + pushed_len); + pages.flush()?; + + Ok(true) + } + + #[inline] + fn serialize_changes(&self) -> Result> { + self.serialize_compressed_changes() + } + + #[inline] + fn db(&self) -> Database { + self.base.db() + } + + fn any_stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + >::stamped_write_with_changes(self, stamp) + } + + fn remove(self) -> Result<()> { + Self::remove(self) + } + + fn any_truncate_if_needed_at(&mut self, index: usize) -> Result<()> { + >::truncate_if_needed_at(self, index) + } + + fn any_reset(&mut self) -> Result<()> { + >::reset(self) + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/any_vec.rs b/crates/vecdb/src/variants/compressed/inner/read_write/any_vec.rs new file mode 100644 index 000000000..27944f9be --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/any_vec.rs @@ -0,0 +1,45 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::{super::CompressionStrategy, ReadWriteCompressedVec}; + +impl AnyVec for ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline] + fn version(&self) -> Version { + self.base.version() + } + + #[inline] + fn name(&self) -> &str { + self.base.name() + } + + #[inline] + fn len(&self) -> usize { + self.base.len() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + vec![self.base.index_to_name(), self.pages_region_name()] + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/mod.rs b/crates/vecdb/src/variants/compressed/inner/read_write/mod.rs new file mode 100644 index 000000000..cbba11ea6 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/mod.rs @@ -0,0 +1,313 @@ +use std::{marker::PhantomData, sync::Arc}; + +use log::info; +use parking_lot::RwLock; +use rawdb::{Reader, likely, unlikely}; + +mod any_stored_vec; +mod any_vec; +mod readable; +mod rollback; +mod typed; +mod writable; + +use crate::{ + AnyStoredVec, AnyVec, Error, Format, ImportOptions, MMAP_CROSSOVER_BYTES, ReadWriteBaseVec, + Result, VecIndex, VecValue, Version, WritableVec, vec_region_name_with, +}; + +use super::{CompressionStrategy, Pages, ReadOnlyCompressedVec}; + +/// Maximum size in bytes of a single uncompressed page (16 KiB). +/// Smaller pages reduce memory overhead during decompression and improve +/// random access performance, while larger pages compress more efficiently. +/// 16 KiB balances these trade-offs for typical workloads. +pub const MAX_UNCOMPRESSED_PAGE_SIZE: usize = 16 * 1024; + +const VERSION: Version = Version::new(3); + +/// Inner implementation for compressed storage vectors. +/// Parameterized by compression strategy to support different compression algorithms. +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct ReadWriteCompressedVec { + pub(super) base: ReadWriteBaseVec, + pub(super) pages: Arc>, + _strategy: PhantomData, +} + +impl ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + pub(super) const PER_PAGE: usize = MAX_UNCOMPRESSED_PAGE_SIZE / Self::SIZE_OF_T; + + pub fn read_only_clone(&self) -> ReadOnlyCompressedVec { + ReadOnlyCompressedVec::new(self.base.read_only_base(), Arc::clone(&self.pages)) + } + + /// # Warning + /// + /// This will DELETE all existing data on format/version errors. Use with caution. + pub fn forced_import_with(mut options: ImportOptions, format: Format) -> Result { + options.version = options.version + VERSION; + let res = Self::import_with(options, format); + match res { + Err(Error::WrongEndian) + | Err(Error::WrongLength { .. }) + | Err(Error::DifferentFormat { .. }) + | Err(Error::DifferentVersion { .. }) => { + info!("Resetting {}...", options.name); + options + .db + .remove_region_if_exists(&vec_region_name_with::(options.name))?; + options + .db + .remove_region_if_exists(&Self::pages_region_name_with(options.name))?; + Self::import_with(options, format) + } + _ => res, + } + } + + #[inline] + pub fn import_with(mut options: ImportOptions, format: Format) -> Result { + options.version = options.version + VERSION; + let db = options.db; + let name = options.name; + + let base = ReadWriteBaseVec::import(options, format)?; + + let pages = Pages::import(db, &Self::pages_region_name_with(name))?; + + let mut this = Self { + base, + pages: Arc::new(RwLock::new(pages)), + _strategy: PhantomData, + }; + + let len = this.real_stored_len(); + *this.base.mut_prev_stored_len() = len; + this.base.update_stored_len(len); + + Ok(this) + } + + #[inline] + pub fn decode_page(&self, page_index: usize, reader: &Reader) -> Result> { + Self::decode_page_with(self.stored_len(), page_index, reader, &self.pages.read()) + } + + #[inline] + pub(crate) fn decode_page_with( + stored_len: usize, + page_index: usize, + reader: &Reader, + pages: &Pages, + ) -> Result> { + let index = Self::page_index_to_index(page_index); + + if unlikely(index >= stored_len) { + return Err(Error::IndexTooHigh { + index, + len: stored_len, + name: "page".to_string(), + }); + } + if unlikely(page_index >= pages.len()) { + return Err(Error::ExpectVecToHaveIndex); + } + + // SAFETY: We checked page_index < pages.len() above + let page = pages + .get(page_index) + .expect("page should exist after bounds check"); + let data = reader.unchecked_read(page.start as usize, page.bytes as usize); + S::decode_page(data, page) + } + + #[inline] + pub(super) fn compress_page(chunk: &[T]) -> Result> { + debug_assert!( + chunk.len() <= Self::PER_PAGE, + "chunk length {} exceeds PER_PAGE {}", + chunk.len(), + Self::PER_PAGE + ); + + S::compress(chunk) + } + + #[inline(always)] + pub(crate) fn index_to_page_index(index: usize) -> usize { + index / Self::PER_PAGE + } + + #[inline(always)] + pub(crate) fn page_index_to_index(page_index: usize) -> usize { + page_index * Self::PER_PAGE + } + + /// Reads stored page data into a buffer. Used by both ReadWrite and ReadOnly read_into_at. + #[inline(always)] + pub(crate) fn read_stored_pages_into( + reader: &Reader, + pages: &Pages, + from: usize, + to: usize, + buf: &mut Vec, + ) { + let start_page = Self::index_to_page_index(from); + let end_page = Self::index_to_page_index(to - 1); + for page_idx in start_page..=end_page { + let page_start = Self::page_index_to_index(page_idx); + let page = pages + .get(page_idx) + .expect("page should exist after bounds check"); + let data = reader.unchecked_read(page.start as usize, page.bytes as usize); + let values_count = page.values_count() as usize; + let local_from = from.saturating_sub(page_start); + let local_to = (to - page_start).min(values_count); + + if !page.is_raw() && likely(local_from == 0) { + let before = buf.len(); + S::decompress_append(data, values_count, buf) + .expect("decompression failed in read_into_at"); + buf.truncate(before + local_to); + } else { + let mut page_buf = Vec::with_capacity(values_count); + S::decode_page_into(data, page, &mut page_buf) + .expect("page decode failed in read_into_at"); + buf.extend_from_slice(&page_buf[local_from..local_to]); + } + } + } + + pub(crate) fn pages_region_name(&self) -> String { + Self::pages_region_name_with(self.name()) + } + + fn pages_region_name_with(name: &str) -> String { + format!("{}_pages", vec_region_name_with::(name)) + } + + pub fn remove(self) -> Result<()> { + self.base.remove()?; + + let pages = Arc::try_unwrap(self.pages).map_err(|_| Error::PagesStillReferenced)?; + pages.into_inner().remove()?; + + Ok(()) + } + + #[inline] + pub fn reserve_pushed(&mut self, additional: usize) { + self.base.reserve_pushed(additional); + } + + #[inline] + pub(crate) fn create_reader(&self) -> Reader { + self.base.region().create_reader() + } + + #[inline] + pub(crate) fn pages(&self) -> &Arc> { + &self.pages + } + + pub(crate) fn collect_stored_range(&self, from: usize, to: usize) -> Result> { + if from >= to { + return Ok(vec![]); + } + + let reader = self.create_reader(); + let pages = self.pages.read(); + let real_len = pages.stored_len(Self::PER_PAGE); + let to = to.min(real_len); + if from >= to { + return Ok(vec![]); + } + + let mut result = Vec::with_capacity(to - from); + let start_page = Self::index_to_page_index(from); + let end_page = Self::index_to_page_index(to - 1); + + for page_idx in start_page..=end_page { + let page_start = Self::page_index_to_index(page_idx); + let decoded = Self::decode_page_with(real_len, page_idx, &reader, &pages)?; + let local_from = from.saturating_sub(page_start); + let local_to = (to - page_start).min(decoded.len()); + result.extend_from_slice(&decoded[local_from..local_to]); + } + + Ok(result) + } + + #[inline] + pub fn fold_stored_io B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + let stored_len = self.stored_len(); + let from = from.min(stored_len); + let to = to.min(stored_len); + if from >= to { + return init; + } + crate::CompressedIoSource::new(self, from, to).fold(init, f) + } + + #[inline] + pub fn fold_stored_mmap B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + let stored_len = self.stored_len(); + let from = from.min(stored_len); + let to = to.min(stored_len); + if from >= to { + return init; + } + crate::CompressedMmapSource::new(self, from, to).fold(init, f) + } + + #[inline(always)] + pub(super) fn fold_source B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + let range_bytes = (to - from) * Self::SIZE_OF_T; + if range_bytes > MMAP_CROSSOVER_BYTES { + crate::CompressedIoSource::new(self, from, to).fold(init, f) + } else { + crate::CompressedMmapSource::new(self, from, to).fold(init, f) + } + } + + #[inline(always)] + pub(super) fn try_fold_source std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result { + let range_bytes = (to - from) * Self::SIZE_OF_T; + if range_bytes > MMAP_CROSSOVER_BYTES { + crate::CompressedIoSource::new(self, from, to).try_fold(init, f) + } else { + crate::CompressedMmapSource::new(self, from, to).try_fold(init, f) + } + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/readable.rs b/crates/vecdb/src/variants/compressed/inner/read_write/readable.rs new file mode 100644 index 000000000..3f4c13998 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/readable.rs @@ -0,0 +1,104 @@ +use crate::{AnyStoredVec, READ_CHUNK_SIZE, ReadableVec, VecIndex, VecValue}; + +use super::{super::CompressionStrategy, ReadWriteCompressedVec}; + +impl ReadableVec for ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline(always)] + fn cursor_chunk_size(&self) -> usize { + Self::PER_PAGE * READ_CHUNK_SIZE.div_ceil(Self::PER_PAGE) + } + + #[inline(always)] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return; + } + + buf.reserve(to - from); + let stored_len = self.stored_len(); + + if from < stored_len { + let stored_to = to.min(stored_len); + let reader = self.create_reader(); + let pages = self.pages.read(); + Self::read_stored_pages_into(&reader, &pages, from, stored_to, buf); + } + + if to > stored_len { + let push_from = from.max(stored_len); + let pushed = self.base.pushed(); + let start = push_from - stored_len; + let end = (to - stored_len).min(pushed.len()); + buf.extend_from_slice(&pushed[start..end]); + } + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + self.fold_range_at(from, to, (), |(), v| f(v)); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return init; + } + + let stored_len = self.stored_len(); + + if to <= stored_len { + return self.fold_source(from, to, init, f); + } + + let mut acc = init; + if from < stored_len { + acc = self.fold_source(from, stored_len, acc, &mut f); + } + self.base.fold_pushed(from, to, acc, f) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return Ok(init); + } + + let stored_len = self.stored_len(); + + if to <= stored_len { + return self.try_fold_source(from, to, init, f); + } + + let mut acc = init; + if from < stored_len { + acc = self.try_fold_source(from, stored_len, acc, &mut f)?; + } + self.base.try_fold_pushed(from, to, acc, f) + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/rollback.rs b/crates/vecdb/src/variants/compressed/inner/read_write/rollback.rs new file mode 100644 index 000000000..2eb6a4ede --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/rollback.rs @@ -0,0 +1,45 @@ +use crate::{ + AnyStoredVec, ChangeCursor, ReadWriteBaseVec, Result, VecIndex, VecValue, WritableVec, +}; + +use super::{super::CompressionStrategy, ReadWriteCompressedVec}; + +impl ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + pub(super) fn serialize_compressed_changes(&self) -> Result> { + self.base.serialize_changes( + Self::SIZE_OF_T, + |from, to| self.collect_stored_range(from, to), + |vals, buf| { + for v in vals { + S::write_to_vec(v, buf); + } + }, + ) + } + + pub(super) fn deserialize_then_undo_changes(&mut self, bytes: &[u8]) -> Result<()> { + let mut c = ChangeCursor::new(bytes); + let change = + ReadWriteBaseVec::::parse_change_data(&mut c, Self::SIZE_OF_T, |b| S::read(b))?; + + // No overlay map: truncated values ride in `pushed` and `stored_len` + // is clamped to where disk still agrees with the rolled-back state. + let (stored_len, pushed) = if change.truncated_values.is_empty() { + (change.prev_stored_len, change.prev_pushed) + } else { + let agree_at = change.truncated_start.min(self.real_stored_len()); + let mut buf = change.truncated_values; + buf.extend(change.prev_pushed); + (agree_at, buf) + }; + self.base + .apply_rollback(change.prev_stamp, stored_len, pushed); + + Ok(()) + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/typed.rs b/crates/vecdb/src/variants/compressed/inner/read_write/typed.rs new file mode 100644 index 000000000..06550408a --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/typed.rs @@ -0,0 +1,13 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::{super::CompressionStrategy, ReadWriteCompressedVec}; + +impl TypedVec for ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/compressed/inner/read_write/writable.rs b/crates/vecdb/src/variants/compressed/inner/read_write/writable.rs new file mode 100644 index 000000000..276af8351 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/read_write/writable.rs @@ -0,0 +1,69 @@ +use std::{collections::BTreeMap, path::PathBuf}; + +use crate::{AnyStoredVec, Result, Stamp, VecIndex, VecValue, WritableVec}; + +use super::{super::CompressionStrategy, ReadWriteCompressedVec}; + +impl WritableVec for ReadWriteCompressedVec +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + #[inline] + fn push(&mut self, value: T) { + self.base.mut_pushed().push(value); + } + + #[inline] + fn pushed(&self) -> &[T] { + self.base.pushed() + } + + fn truncate_if_needed_at(&mut self, index: usize) -> Result<()> { + if self.base.truncate_pushed(index) { + self.base.update_stored_len(index); + } + Ok(()) + } + + fn reset(&mut self) -> Result<()> { + self.pages.write().reset(); + self.truncate_if_needed_at(0)?; + self.base.reset_base() + } + + fn reset_unsaved(&mut self) { + self.base.reset_unsaved_base(); + } + + fn is_dirty(&self) -> bool { + !self.base.pushed().is_empty() + } + + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + if self.base.saved_stamped_changes() == 0 { + return self.stamped_write(stamp); + } + + let data = self.serialize_changes()?; + self.base.save_change_file(stamp, &data)?; + self.stamped_write(stamp)?; + self.base.save_prev(); + + Ok(()) + } + + fn rollback(&mut self) -> Result<()> { + let bytes = self.base.read_current_change_file()?; + self.deserialize_then_undo_changes(&bytes) + } + + fn find_rollback_files(&self) -> Result> { + self.base.find_rollback_files() + } + + fn save_rollback_state(&mut self) { + self.base.save_prev_for_rollback(); + } +} diff --git a/crates/vecdb/src/variants/compressed/inner/strategy.rs b/crates/vecdb/src/variants/compressed/inner/strategy.rs new file mode 100644 index 000000000..579e9e2e5 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/inner/strategy.rs @@ -0,0 +1,128 @@ +use rawdb::likely; + +use crate::{Error, Result, ValueStrategy}; + +use super::Page; + +/// Trait for compression strategies used by ReadWriteCompressedVec. +pub trait CompressionStrategy: ValueStrategy { + /// Compress a slice of values into bytes. + fn compress(values: &[T]) -> Result>; + + /// Decompress bytes into a vector of values. + fn decompress(bytes: &[u8], expected_len: usize) -> Result>; + + /// Decompress bytes into an existing buffer (replace semantics). + /// Implementations should reuse dst's allocation when possible (see PcodecStrategy). + /// Default implementation replaces dst with a new Vec from `decompress`. + #[inline] + fn decompress_into(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + *dst = Self::decompress(bytes, expected_len)?; + Ok(()) + } + + /// Decompress bytes, appending to dst without clearing it. + /// Default implementation decompresses then appends via extend. + #[inline] + fn decompress_append(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + let tmp = Self::decompress(bytes, expected_len)?; + dst.extend(tmp); + Ok(()) + } + + /// Decode page data (raw or compressed) into a new Vec. + #[inline] + fn decode_page(data: &[u8], page: &Page) -> Result> { + let n = page.values_count() as usize; + if page.is_raw() { + Self::bytes_to_values(data, n) + } else { + let vec = Self::decompress(data, n)?; + if likely(vec.len() == n) { + return Ok(vec); + } + Err(Error::DecompressionMismatch { + expected_len: n, + actual_len: vec.len(), + }) + } + } + + /// Decode page data (raw or compressed) into an existing buffer (replace semantics). + #[inline] + fn decode_page_into(data: &[u8], page: &Page, dst: &mut Vec) -> Result<()> { + let n = page.values_count() as usize; + if page.is_raw() { + Self::bytes_to_values_into(data, n, dst) + } else { + Self::decompress_into(data, n, dst) + } + } + + /// Serializes a slice of values to bytes. + #[inline] + fn values_to_bytes(values: &[T]) -> Vec { + let byte_len = size_of_val(values); + let mut bytes = Vec::with_capacity(byte_len); + if Self::IS_NATIVE_LAYOUT { + unsafe { + std::ptr::copy_nonoverlapping( + values.as_ptr() as *const u8, + bytes.as_mut_ptr(), + byte_len, + ); + bytes.set_len(byte_len); + } + } else { + for v in values { + Self::write_to_vec(v, &mut bytes); + } + } + bytes + } + + /// Deserializes bytes to a vector of values, validating the expected length. + #[inline] + fn bytes_to_values(bytes: &[u8], expected_len: usize) -> Result> { + let mut vec = Vec::with_capacity(expected_len); + Self::bytes_to_values_into(bytes, expected_len, &mut vec)?; + Ok(vec) + } + + /// Deserializes bytes into an existing buffer, reusing its allocation. + #[inline] + fn bytes_to_values_into(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + let expected_bytes = expected_len * size_of::(); + dst.clear(); + dst.reserve(expected_len); + if Self::IS_NATIVE_LAYOUT { + if likely(bytes.len() >= expected_bytes) { + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + dst.as_mut_ptr() as *mut u8, + expected_bytes, + ); + dst.set_len(expected_len); + } + return Ok(()); + } + } else { + for chunk in bytes.chunks_exact(size_of::()) { + dst.push(Self::read(chunk)?); + } + if likely(dst.len() == expected_len) { + return Ok(()); + } + } + + Err(Error::DecompressionMismatch { + expected_len, + actual_len: if Self::IS_NATIVE_LAYOUT { + bytes.len() / size_of::() + } else { + dst.len() + }, + }) + } +} diff --git a/crates/vecdb/src/variants/compressed/lz4/mod.rs b/crates/vecdb/src/variants/compressed/lz4/mod.rs new file mode 100644 index 000000000..9229c7e16 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/lz4/mod.rs @@ -0,0 +1,33 @@ +use crate::{Format, ReadOnlyCompressedVec, ReadWriteCompressedVec, impl_vec_wrapper}; + +mod strategy; +mod value; + +pub use strategy::*; +pub use value::*; + +/// Compressed storage using LZ4 for speed-optimized general-purpose compression. +/// +/// LZ4 prioritizes compression/decompression speed over ratio, making it ideal +/// for workloads where CPU time matters more than storage space. +/// +/// # Performance Characteristics +/// - Extremely fast compression/decompression (hundreds of MB/s) +/// - Moderate compression ratios (typically 2-3x) +/// - Works well with any data type +/// +/// # When to Use +/// - Speed is more important than storage savings +/// - Mixed data types (not just numbers) +/// - Need compression but can't afford CPU overhead +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct LZ4Vec(ReadWriteCompressedVec>); + +impl_vec_wrapper!( + LZ4Vec, + ReadWriteCompressedVec>, + LZ4VecValue, + Format::LZ4, + ReadOnlyCompressedVec> +); diff --git a/crates/vecdb/src/variants/compressed/lz4/strategy.rs b/crates/vecdb/src/variants/compressed/lz4/strategy.rs new file mode 100644 index 000000000..51cc64a9e --- /dev/null +++ b/crates/vecdb/src/variants/compressed/lz4/strategy.rs @@ -0,0 +1,32 @@ +use std::marker::PhantomData; + +use lz4_flex::{compress_prepend_size, decompress_size_prepended}; + +use crate::{Result, impl_bytes_value_strategy}; + +use super::{super::inner::CompressionStrategy, value::LZ4VecValue}; + +/// LZ4 compression strategy for fast compression/decompression. +#[derive(Debug, Clone, Copy)] +pub struct LZ4Strategy(PhantomData); + +impl_bytes_value_strategy!(LZ4Strategy, LZ4VecValue); + +impl CompressionStrategy for LZ4Strategy +where + T: LZ4VecValue, +{ + fn compress(values: &[T]) -> Result> { + Ok(compress_prepend_size(&Self::values_to_bytes(values))) + } + + fn decompress(bytes: &[u8], expected_len: usize) -> Result> { + let decompressed = decompress_size_prepended(bytes)?; + Self::bytes_to_values(&decompressed, expected_len) + } + + fn decompress_into(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + let decompressed = decompress_size_prepended(bytes)?; + Self::bytes_to_values_into(&decompressed, expected_len, dst) + } +} diff --git a/crates/vecdb/src/variants/compressed/lz4/value.rs b/crates/vecdb/src/variants/compressed/lz4/value.rs new file mode 100644 index 000000000..00b885280 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/lz4/value.rs @@ -0,0 +1,11 @@ +use crate::{Bytes, VecValue}; + +/// Value trait for LZ4Vec. +/// Extends VecValue with Bytes trait for byte serialization. +pub trait LZ4VecValue +where + Self: VecValue + Bytes, +{ +} + +impl LZ4VecValue for T where T: VecValue + Bytes {} diff --git a/crates/vecdb/src/variants/compressed/mod.rs b/crates/vecdb/src/variants/compressed/mod.rs new file mode 100644 index 000000000..1c991dfa8 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/mod.rs @@ -0,0 +1,18 @@ +mod inner; +#[cfg(feature = "lz4")] +mod lz4; +#[cfg(feature = "pco")] +mod pco; +mod sources; +#[cfg(feature = "zstd")] +mod zstd; + +pub(crate) use inner::*; +pub use inner::{CompressionStrategy, ReadOnlyCompressedVec}; +#[cfg(feature = "lz4")] +pub use lz4::*; +#[cfg(feature = "pco")] +pub use pco::*; +pub(crate) use sources::*; +#[cfg(feature = "zstd")] +pub use zstd::*; diff --git a/crates/vecdb/src/variants/compressed/pco/mod.rs b/crates/vecdb/src/variants/compressed/pco/mod.rs new file mode 100644 index 000000000..741cc6c78 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/mod.rs @@ -0,0 +1,36 @@ +use crate::{Format, ReadOnlyCompressedVec, ReadWriteCompressedVec, impl_vec_wrapper}; + +mod strategy; +mod r#trait; +mod value; + +pub use strategy::*; +pub use r#trait::*; +pub use value::*; + +/// Compressed storage using Pcodec for optimal numeric data compression. +/// +/// Pcodec (Pco) provides the best compression ratios for numerical sequences +/// through specialized quantization and encoding. Ideal for time-series, scientific +/// data, and any workload dominated by numeric values. +/// +/// # Performance Characteristics +/// - Best-in-class compression for numeric types (often 2-10x better than LZ4/Zstd) +/// - Sequential access optimized (values compressed in pages) +/// - Random access possible but slower than raw formats +/// +/// # When to Use +/// - Numeric data dominates (integers, floats) +/// - Storage space is critical +/// - Sequential access patterns are common +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct PcoVec(ReadWriteCompressedVec>); + +impl_vec_wrapper!( + PcoVec, + ReadWriteCompressedVec>, + PcoVecValue, + Format::Pco, + ReadOnlyCompressedVec> +); diff --git a/crates/vecdb/src/variants/compressed/pco/strategy.rs b/crates/vecdb/src/variants/compressed/pco/strategy.rs new file mode 100644 index 000000000..2cebf0796 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/strategy.rs @@ -0,0 +1,81 @@ +use std::marker::PhantomData; + +use pco::ChunkConfig; +use pco::standalone::{simple_compress, simple_decompress, simple_decompress_into}; + +use crate::{Error, Result, impl_bytes_value_strategy, likely}; + +use super::{ + super::inner::CompressionStrategy, + value::{AsInnerSlice, FromInnerSlice, PcoVecValue}, +}; + +/// Returns the default ChunkConfig for pcodec compression. +fn chunk_config() -> ChunkConfig { + ChunkConfig::default().with_enable_8_bit(true) +} + +/// Pcodec compression strategy for numerical data. +#[derive(Debug, Clone, Copy)] +pub struct PcodecStrategy(PhantomData); + +impl_bytes_value_strategy!(PcodecStrategy, PcoVecValue); + +impl CompressionStrategy for PcodecStrategy +where + T: PcoVecValue, +{ + fn compress(values: &[T]) -> Result> { + Ok(simple_compress(values.as_inner_slice(), &chunk_config())?) + } + + fn decompress(bytes: &[u8], expected_len: usize) -> Result> { + let vec: Vec = simple_decompress(bytes)?; + let vec = T::from_inner_slice(vec); + + if likely(vec.len() == expected_len) { + return Ok(vec); + } + + Err(Error::DecompressionMismatch { + expected_len, + actual_len: vec.len(), + }) + } + + #[inline] + fn decompress_into(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + dst.clear(); + Self::decompress_append(bytes, expected_len, dst) + } + + #[inline] + fn decompress_append(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + dst.reserve(expected_len); + let old_len = dst.len(); + + // SAFETY: MaybeUninit has the same layout as T::NumberType. + // simple_decompress_into will initialize the memory, and we only set_len + // after initialization succeeds. + let spare = dst.spare_capacity_mut(); + let slice = unsafe { + std::slice::from_raw_parts_mut(spare.as_mut_ptr().cast::(), expected_len) + }; + + let progress = simple_decompress_into(bytes, slice)?; + + // SAFETY: simple_decompress_into initialized progress.n_processed elements. + unsafe { + dst.set_len(old_len + progress.n_processed); + } + + if likely(progress.n_processed == expected_len) { + return Ok(()); + } + + Err(Error::DecompressionMismatch { + expected_len, + actual_len: progress.n_processed, + }) + } +} diff --git a/crates/vecdb/src/variants/compressed/pco/trait.rs b/crates/vecdb/src/variants/compressed/pco/trait.rs new file mode 100644 index 000000000..c0fff0812 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/trait.rs @@ -0,0 +1,13 @@ +use pco::data_types::Number; + +/// Witnesses that `Self` has the same memory layout as `T`, enabling the +/// zero-copy cast between `[Self]` and `[T::NumberType]` used by pco's encoder. +pub trait TransparentPco {} + +/// Binds a vec value type to its underlying pco `Number` representation. +pub trait Pco +where + Self: TransparentPco, +{ + type NumberType: Number; +} diff --git a/crates/vecdb/src/variants/compressed/pco/value/as_inner.rs b/crates/vecdb/src/variants/compressed/pco/value/as_inner.rs new file mode 100644 index 000000000..bffa48853 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/value/as_inner.rs @@ -0,0 +1,33 @@ +use std::mem::{align_of, size_of}; + +use pco::data_types::Number; + +use super::PcoVecValue; + +/// Convert a slice of PcoVecValue to a slice of the underlying Number type. +/// +/// # Safety +/// This trait uses unsafe pointer casting that relies on compile-time size/alignment checks. +/// The const assertions ensure T and T::NumberType have identical layout. +pub trait AsInnerSlice +where + T: Number, +{ + const _SIZE_CHECK: (); + const _ALIGN_CHECK: (); + + fn as_inner_slice(&self) -> &[T]; +} + +impl AsInnerSlice for [T] +where + T: PcoVecValue, +{ + const _SIZE_CHECK: () = assert!(size_of::() == size_of::()); + const _ALIGN_CHECK: () = assert!(align_of::() == align_of::()); + + fn as_inner_slice(&self) -> &[T::NumberType] { + // SAFETY: Compile-time assertions ensure T and T::NumberType have identical layout + unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T::NumberType, self.len()) } + } +} diff --git a/crates/vecdb/src/variants/compressed/pco/value/as_inner_mut.rs b/crates/vecdb/src/variants/compressed/pco/value/as_inner_mut.rs new file mode 100644 index 000000000..13ed728b5 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/value/as_inner_mut.rs @@ -0,0 +1,35 @@ +use std::mem::{align_of, size_of}; + +use pco::data_types::Number; + +use super::PcoVecValue; + +/// Convert a mutable slice of PcoVecValue to a mutable slice of the underlying Number type. +/// +/// # Safety +/// This trait uses unsafe pointer casting that relies on compile-time size/alignment checks. +/// The const assertions ensure T and T::NumberType have identical layout. +pub trait AsInnerSliceMut +where + T: Number, +{ + const _SIZE_CHECK: (); + const _ALIGN_CHECK: (); + + fn as_inner_slice_mut(&mut self) -> &mut [T]; +} + +impl AsInnerSliceMut for [T] +where + T: PcoVecValue, +{ + const _SIZE_CHECK: () = assert!(size_of::() == size_of::()); + const _ALIGN_CHECK: () = assert!(align_of::() == align_of::()); + + fn as_inner_slice_mut(&mut self) -> &mut [T::NumberType] { + // SAFETY: Compile-time assertions ensure T and T::NumberType have identical layout + unsafe { + std::slice::from_raw_parts_mut(self.as_mut_ptr() as *mut T::NumberType, self.len()) + } + } +} diff --git a/crates/vecdb/src/variants/compressed/pco/value/from_inner.rs b/crates/vecdb/src/variants/compressed/pco/value/from_inner.rs new file mode 100644 index 000000000..a68c378fa --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/value/from_inner.rs @@ -0,0 +1,30 @@ +use std::mem::{ManuallyDrop, align_of, size_of}; + +use super::PcoVecValue; + +/// Convert a Vec of Number type to a Vec of PcoVecValue. +/// +/// # Safety +/// This trait uses unsafe pointer casting that relies on compile-time size/alignment checks. +/// The const assertions ensure T and T::NumberType have identical layout. +pub trait FromInnerSlice { + const _SIZE_CHECK: (); + const _ALIGN_CHECK: (); + + fn from_inner_slice(slice: Vec) -> Vec + where + Self: Sized; +} + +impl FromInnerSlice for T +where + T: PcoVecValue, +{ + const _SIZE_CHECK: () = assert!(size_of::() == size_of::()); + const _ALIGN_CHECK: () = assert!(align_of::() == align_of::()); + + fn from_inner_slice(vec: Vec) -> Vec { + let mut vec = ManuallyDrop::new(vec); + unsafe { Vec::from_raw_parts(vec.as_mut_ptr() as *mut T, vec.len(), vec.capacity()) } + } +} diff --git a/crates/vecdb/src/variants/compressed/pco/value/mod.rs b/crates/vecdb/src/variants/compressed/pco/value/mod.rs new file mode 100644 index 000000000..56a84f3d4 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/pco/value/mod.rs @@ -0,0 +1,28 @@ +mod as_inner; +mod as_inner_mut; +mod from_inner; + +pub use as_inner::AsInnerSlice; +pub use as_inner_mut::AsInnerSliceMut; +pub use from_inner::FromInnerSlice; + +use crate::{BytesVecValue, Pco, TransparentPco}; + +/// Marker trait for values storable in a `PcoVec`: must be `Copy`, `Pco`, +/// and serializable via the `Bytes` path used by `BytesVec`. +pub trait PcoVecValue: Pco + BytesVecValue + Copy {} + +impl PcoVecValue for T where T: Pco + BytesVecValue + Copy {} + +macro_rules! impl_stored_compressed { + ($($t:ty),*) => { + $( + impl TransparentPco<$t> for $t {} + impl Pco for $t { + type NumberType = $t; + } + )* + }; +} + +impl_stored_compressed!(u8, u16, u32, u64, i8, i16, i32, i64, f32, f64); diff --git a/crates/vecdb/src/variants/compressed/sources/io.rs b/crates/vecdb/src/variants/compressed/sources/io.rs new file mode 100644 index 000000000..c48eeb00b --- /dev/null +++ b/crates/vecdb/src/variants/compressed/sources/io.rs @@ -0,0 +1,217 @@ +use std::{ + fs::File, + io::{Read, Seek, SeekFrom}, + sync::Arc, +}; + +use parking_lot::{RwLock, RwLockReadGuard}; +use rawdb::{Region, RegionMetadata}; + +use crate::{AnyStoredVec, BUFFER_SIZE, Pages, VecIndex, VecValue, unlikely}; + +use super::super::inner::{ + CompressionStrategy, MAX_UNCOMPRESSED_PAGE_SIZE, Page, ReadWriteCompressedVec, +}; + +/// Buffered file I/O source for reading stored compressed data. +/// +/// Uses dedicated file handle for sequential reads (better OS readahead than mmap). +/// Only sees stored (persisted) values. Consumed by fold/try_fold/for_each. +pub struct CompressedIoSource<'a, I, T, S> { + file: File, + file_position: u64, + region_start: u64, + buffer: Vec, + buffer_len: usize, + buffer_start_offset: u64, + decoded_values: Vec, + decoded_page_index: usize, + pages: RwLockReadGuard<'a, Pages>, + index: usize, + end_index: usize, + _region_lock: RwLockReadGuard<'a, RegionMetadata>, + _marker: std::marker::PhantomData<(I, T, S)>, +} + +impl<'a, I, T, S> CompressedIoSource<'a, I, T, S> +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + const SIZE_OF_T: usize = size_of::(); + const PER_PAGE: usize = MAX_UNCOMPRESSED_PAGE_SIZE / Self::SIZE_OF_T; + const NO_PAGE: usize = usize::MAX; + + pub(crate) fn new(vec: &'a ReadWriteCompressedVec, from: usize, to: usize) -> Self { + Self::new_from_parts(vec.region(), vec.pages(), vec.stored_len(), from, to) + } + + pub(crate) fn new_from_parts( + region: &'a Region, + pages: &'a Arc>, + stored_len: usize, + from: usize, + to: usize, + ) -> Self { + let region_lock = region.meta(); + let region_start = region_lock.start() as u64; + let file = region.open_db_read_only_file().expect("open file"); + let pages = pages.read(); + let from = from.min(stored_len); + let to = to.min(stored_len); + + Self { + file, + file_position: 0, + region_start, + buffer: vec![0; BUFFER_SIZE], + buffer_len: 0, + buffer_start_offset: 0, + decoded_values: Vec::with_capacity(Self::PER_PAGE), + decoded_page_index: Self::NO_PAGE, + pages, + index: from, + end_index: to, + _region_lock: region_lock, + _marker: std::marker::PhantomData, + } + } + + #[inline(always)] + fn ensure_page_decoded(&mut self, page_index: usize) -> Option<()> { + if unlikely(self.decoded_page_index != page_index) { + self.decode_page(page_index)?; + } + Some(()) + } + + fn refill_buffer(&mut self, starting_page_index: usize) -> Option<()> { + let start_page = self.pages.get(starting_page_index)?; + let start_offset = start_page.start; + + let last_needed_page = if self.end_index == 0 { + 0 + } else { + (self.end_index - 1) / Self::PER_PAGE + }; + let max_page = last_needed_page.min(self.pages.len().saturating_sub(1)); + + let mut total_bytes = 0usize; + for i in starting_page_index..=max_page { + let page = self.pages.get(i)?; + let page_bytes = page.bytes as usize; + if total_bytes + page_bytes > BUFFER_SIZE { + break; + } + total_bytes += page_bytes; + } + + if total_bytes == 0 { + return None; + } + + let absolute_offset = self.region_start + start_offset; + if self.file_position != absolute_offset { + self.file_position = absolute_offset; + self.file.seek(SeekFrom::Start(absolute_offset)).unwrap(); + } + + self.file + .read_exact(&mut self.buffer[..total_bytes]) + .unwrap(); + self.buffer_len = total_bytes; + self.buffer_start_offset = start_offset; + self.file_position += total_bytes as u64; + + Some(()) + } + + fn decode_from_buffer(&mut self, page_index: usize, page: Page) -> Option<()> { + let in_buffer_offset = (page.start - self.buffer_start_offset) as usize; + let data = &self.buffer[in_buffer_offset..in_buffer_offset + page.bytes as usize]; + + S::decode_page_into(data, &page, &mut self.decoded_values).ok()?; + self.decoded_page_index = page_index; + + Some(()) + } + + fn decode_page(&mut self, page_index: usize) -> Option<()> { + if page_index >= self.pages.len() { + return None; + } + + // Copy page metadata to release the borrow on self.pages + let page = *self.pages.get(page_index)?; + + if self.buffer_len > 0 { + let buffer_end_offset = self.buffer_start_offset + self.buffer_len as u64; + + if page.start >= self.buffer_start_offset && page.end() <= buffer_end_offset { + return self.decode_from_buffer(page_index, page); + } + } + + self.refill_buffer(page_index)?; + self.decode_from_buffer(page_index, page) + } + + /// Fold all remaining elements — tight pointer loop per page so LLVM can vectorize. + #[inline(always)] + pub(crate) fn fold B>(mut self, init: B, mut f: F) -> B { + let per_page = Self::PER_PAGE; + let end_index = self.end_index; + let mut page_index = self.index / per_page; + let mut page_start = page_index * per_page; + let mut in_page_offset = self.index - page_start; + let mut accum = init; + while self.index < end_index { + if self.ensure_page_decoded(page_index).is_none() { + break; + } + let page_end = (end_index - page_start).min(self.decoded_values.len()); + let ptr = self.decoded_values.as_ptr(); + let mut i = in_page_offset; + while i < page_end { + // Clone rather than move: decoded_values still owns and drops every value. + accum = f(accum, unsafe { (&*ptr.add(i)).clone() }); + i += 1; + } + self.index = page_start + page_end; + page_index += 1; + page_start += per_page; + in_page_offset = 0; + } + accum + } + + /// Fallible fold with early exit on error. + #[inline(always)] + pub(crate) fn try_fold std::result::Result>( + mut self, + init: B, + mut f: F, + ) -> std::result::Result { + let per_page = Self::PER_PAGE; + let end_index = self.end_index; + let mut page_index = self.index / per_page; + let mut page_start = page_index * per_page; + let mut in_page_offset = self.index - page_start; + let mut accum = init; + while self.index < end_index { + if self.ensure_page_decoded(page_index).is_none() { + break; + } + let page_end = (end_index - page_start).min(self.decoded_values.len()); + for value in &self.decoded_values[in_page_offset..page_end] { + accum = f(accum, value.clone())?; + } + self.index = page_start + page_end; + page_index += 1; + page_start += per_page; + in_page_offset = 0; + } + Ok(accum) + } +} diff --git a/crates/vecdb/src/variants/compressed/sources/mmap.rs b/crates/vecdb/src/variants/compressed/sources/mmap.rs new file mode 100644 index 000000000..0c5972314 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/sources/mmap.rs @@ -0,0 +1,138 @@ +use std::{marker::PhantomData, sync::Arc}; + +use parking_lot::{RwLock, RwLockReadGuard}; +use rawdb::{Reader, Region}; + +use crate::{AnyStoredVec, Pages, VecIndex, VecValue, unlikely}; + +use super::super::inner::{ + CompressionStrategy, MAX_UNCOMPRESSED_PAGE_SIZE, ReadWriteCompressedVec, +}; + +/// Read-only mmap-backed source over a compressed vector. +/// +/// Only sees **stored** (persisted) values. Pages are decoded lazily — +/// only when fold/for_each reaches them. Consumed by fold/try_fold/for_each. +pub struct CompressedMmapSource<'a, I, T, S> { + reader: Reader, + pages: RwLockReadGuard<'a, Pages>, + page_buf: Vec, + page_buf_idx: usize, + pos: usize, + end: usize, + _marker: PhantomData<(I, T, S)>, +} + +impl<'a, I, T, S> CompressedMmapSource<'a, I, T, S> +where + I: VecIndex, + T: VecValue, + S: CompressionStrategy, +{ + const SIZE_OF_T: usize = size_of::(); + const PER_PAGE: usize = MAX_UNCOMPRESSED_PAGE_SIZE / Self::SIZE_OF_T; + const NO_PAGE: usize = usize::MAX; + + pub(crate) fn new(vec: &'a ReadWriteCompressedVec, from: usize, to: usize) -> Self { + Self::new_from_parts(vec.region(), vec.pages(), vec.stored_len(), from, to) + } + + pub(crate) fn new_from_parts( + region: &Region, + pages: &'a Arc>, + stored_len: usize, + from: usize, + to: usize, + ) -> Self { + let from = from.min(stored_len); + let to = to.min(stored_len); + Self { + reader: region.create_reader(), + pages: pages.read(), + page_buf: Vec::with_capacity(Self::PER_PAGE), + page_buf_idx: Self::NO_PAGE, + pos: from, + end: to, + _marker: PhantomData, + } + } + + /// Ensures the page at `page_index` is decoded in `page_buf`. + #[inline(always)] + fn ensure_page_decoded(&mut self, page_index: usize) -> Option<()> { + if unlikely(self.page_buf_idx != page_index) { + self.decode_page_into_buf(page_index)?; + } + Some(()) + } + + /// Decode a page into the internal buffer via mmap. + #[inline(always)] + fn decode_page_into_buf(&mut self, page_index: usize) -> Option<()> { + let page = self.pages.get(page_index)?; + let data = self + .reader + .unchecked_read(page.start as usize, page.bytes as usize); + S::decode_page_into(data, page, &mut self.page_buf).ok()?; + self.page_buf_idx = page_index; + Some(()) + } + + /// Fold all remaining elements — tight pointer loop per page so LLVM can vectorize. + #[inline(always)] + pub(crate) fn fold B>(mut self, init: B, mut f: F) -> B { + let per_page = Self::PER_PAGE; + let end = self.end; + let mut page_index = self.pos / per_page; + let mut page_start = page_index * per_page; + let mut in_page_offset = self.pos - page_start; + let mut accum = init; + while self.pos < end { + if self.ensure_page_decoded(page_index).is_none() { + break; + } + let page_end = (end - page_start).min(self.page_buf.len()); + let ptr = self.page_buf.as_ptr(); + let mut i = in_page_offset; + while i < page_end { + // Clone rather than move: page_buf still owns and drops every value. + accum = f(accum, unsafe { (&*ptr.add(i)).clone() }); + i += 1; + } + self.pos = page_start + page_end; + page_index += 1; + page_start += per_page; + in_page_offset = 0; + } + accum + } + + /// Fallible fold with early exit on error. + #[inline(always)] + pub(crate) fn try_fold std::result::Result>( + mut self, + init: B, + mut f: F, + ) -> std::result::Result { + let per_page = Self::PER_PAGE; + let end = self.end; + let mut page_index = self.pos / per_page; + let mut page_start = page_index * per_page; + let mut in_page_offset = self.pos - page_start; + let mut accum = init; + while self.pos < end { + if self.ensure_page_decoded(page_index).is_none() { + break; + } + let page_end = (end - page_start).min(self.page_buf.len()); + for value in &self.page_buf[in_page_offset..page_end] { + accum = f(accum, value.clone())?; + } + self.pos = page_start + page_end; + page_index += 1; + page_start += per_page; + in_page_offset = 0; + } + Ok(accum) + } +} diff --git a/crates/vecdb/src/variants/compressed/sources/mod.rs b/crates/vecdb/src/variants/compressed/sources/mod.rs new file mode 100644 index 000000000..7033cc5ea --- /dev/null +++ b/crates/vecdb/src/variants/compressed/sources/mod.rs @@ -0,0 +1,5 @@ +mod io; +mod mmap; + +pub(crate) use io::*; +pub(crate) use mmap::*; diff --git a/crates/vecdb/src/variants/compressed/zstd/mod.rs b/crates/vecdb/src/variants/compressed/zstd/mod.rs new file mode 100644 index 000000000..93c5be068 --- /dev/null +++ b/crates/vecdb/src/variants/compressed/zstd/mod.rs @@ -0,0 +1,32 @@ +use crate::{Format, ReadOnlyCompressedVec, ReadWriteCompressedVec, impl_vec_wrapper}; + +mod strategy; +mod value; + +pub use strategy::*; +pub use value::*; + +/// Compressed storage using Zstd for maximum general-purpose compression. +/// +/// Zstd (Zstandard) provides the best compression ratios among general-purpose +/// algorithms, with good decompression speed. Ideal when storage is expensive. +/// +/// # Performance Characteristics +/// - Highest compression ratios (typically 3-5x, better than LZ4) +/// - Fast decompression (slower compression than LZ4) +/// - Works well with any data type +/// +/// # When to Use +/// - Storage space is expensive +/// - Can tolerate slower compression (decompression is fast) +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct ZstdVec(ReadWriteCompressedVec>); + +impl_vec_wrapper!( + ZstdVec, + ReadWriteCompressedVec>, + ZstdVecValue, + Format::Zstd, + ReadOnlyCompressedVec> +); diff --git a/crates/vecdb/src/variants/compressed/zstd/strategy.rs b/crates/vecdb/src/variants/compressed/zstd/strategy.rs new file mode 100644 index 000000000..6e242678c --- /dev/null +++ b/crates/vecdb/src/variants/compressed/zstd/strategy.rs @@ -0,0 +1,37 @@ +use std::marker::PhantomData; + +use zstd::{decode_all, encode_all}; + +use crate::{Result, impl_bytes_value_strategy}; + +use super::{super::inner::CompressionStrategy, value::ZstdVecValue}; + +/// Zstd compression level (1-22). Level 3 provides a good balance +/// between compression ratio and speed for most workloads. +const ZSTD_COMPRESSION_LEVEL: i32 = 3; + +/// Zstd compression strategy for high compression ratios. +#[derive(Debug, Clone, Copy)] +pub struct ZstdStrategy(PhantomData); + +impl_bytes_value_strategy!(ZstdStrategy, ZstdVecValue); + +impl CompressionStrategy for ZstdStrategy +where + T: ZstdVecValue, +{ + fn compress(values: &[T]) -> Result> { + let bytes = Self::values_to_bytes(values); + Ok(encode_all(bytes.as_slice(), ZSTD_COMPRESSION_LEVEL)?) + } + + fn decompress(bytes: &[u8], expected_len: usize) -> Result> { + let decompressed = decode_all(bytes)?; + Self::bytes_to_values(&decompressed, expected_len) + } + + fn decompress_into(bytes: &[u8], expected_len: usize, dst: &mut Vec) -> Result<()> { + let decompressed = decode_all(bytes)?; + Self::bytes_to_values_into(&decompressed, expected_len, dst) + } +} diff --git a/crates/vecdb/src/variants/compressed/zstd/value.rs b/crates/vecdb/src/variants/compressed/zstd/value.rs new file mode 100644 index 000000000..e1328175b --- /dev/null +++ b/crates/vecdb/src/variants/compressed/zstd/value.rs @@ -0,0 +1,11 @@ +use crate::{Bytes, VecValue}; + +/// Value trait for ZstdVec. +/// Extends VecValue with Bytes trait for byte serialization. +pub trait ZstdVecValue +where + Self: VecValue + Bytes, +{ +} + +impl ZstdVecValue for T where T: VecValue + Bytes {} diff --git a/crates/vecdb/src/variants/eager/any_stored_vec.rs b/crates/vecdb/src/variants/eager/any_stored_vec.rs new file mode 100644 index 000000000..5436f9b62 --- /dev/null +++ b/crates/vecdb/src/variants/eager/any_stored_vec.rs @@ -0,0 +1,78 @@ +use std::path::PathBuf; + +use rawdb::{Database, Region}; + +use crate::{AnyStoredVec, Header, Result, Stamp, StoredVec, WritableVec}; + +use super::EagerVec; + +impl AnyStoredVec for EagerVec +where + V: StoredVec, +{ + #[inline] + fn db_path(&self) -> PathBuf { + self.0.db_path() + } + + #[inline] + fn region(&self) -> &Region { + self.0.region() + } + + #[inline] + fn header(&self) -> &Header { + self.0.header() + } + + #[inline] + fn mut_header(&mut self) -> &mut Header { + self.0.mut_header() + } + + #[inline] + fn saved_stamped_changes(&self) -> u16 { + self.0.saved_stamped_changes() + } + + #[inline] + fn write(&mut self) -> Result { + self.0.write() + } + + #[inline] + fn stored_len(&self) -> usize { + self.0.stored_len() + } + + #[inline] + fn real_stored_len(&self) -> usize { + self.0.real_stored_len() + } + + #[inline] + fn serialize_changes(&self) -> Result> { + self.0.serialize_changes() + } + + #[inline] + fn db(&self) -> Database { + self.0.db() + } + + fn any_stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + self.0.stamped_write_with_changes(stamp) + } + + fn remove(self) -> Result<()> { + self.0.remove() + } + + fn any_truncate_if_needed_at(&mut self, index: usize) -> Result<()> { + self.truncate_if_needed_at(index) + } + + fn any_reset(&mut self) -> Result<()> { + self.reset() + } +} diff --git a/crates/vecdb/src/variants/eager/any_vec.rs b/crates/vecdb/src/variants/eager/any_vec.rs new file mode 100644 index 000000000..1dd2da599 --- /dev/null +++ b/crates/vecdb/src/variants/eager/any_vec.rs @@ -0,0 +1,43 @@ +use crate::{AnyVec, StoredVec, Version}; + +use super::EagerVec; + +impl AnyVec for EagerVec +where + V: StoredVec, +{ + #[inline] + fn version(&self) -> Version { + self.0.header().computed_version() + } + + #[inline] + fn name(&self) -> &str { + self.0.name() + } + + #[inline] + fn len(&self) -> usize { + self.0.len() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + self.0.index_type_to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + self.0.value_type_to_size_of() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + self.0.value_type_to_string() + } + + #[inline] + fn region_names(&self) -> Vec { + self.0.region_names() + } +} diff --git a/crates/vecdb/src/variants/eager/compute/aggregates.rs b/crates/vecdb/src/variants/eager/compute/aggregates.rs new file mode 100644 index 000000000..9c03b7d23 --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/aggregates.rs @@ -0,0 +1,470 @@ +use std::ops::Add; + +use crate::{ + AnyVec, CheckedSub, Error, Exit, ReadableVec, Result, SaturatingAdd, StoredVec, VecIndex, + VecValue, Version, WritableVec, unlikely, +}; + +use super::super::EagerVec; + +impl EagerVec +where + V: StoredVec, +{ + fn compute_aggregate_of_others( + &mut self, + max_from: V::I, + others: &[&O], + exit: &Exit, + aggregate: F, + ) -> Result<()> + where + O: ReadableVec, + F: Fn(&[Vec], usize) -> V::T, + { + if others.is_empty() { + return Err(Error::InvalidArgument( + "others must have at least one element", + )); + } + + self.compute_init( + others.iter().map(|v| v.version()).sum(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = others.iter().map(|v| v.len()).min().unwrap(); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let batches: Vec> = others + .iter() + .map(|v| v.collect_range_at(skip, end)) + .collect(); + + for j in 0..(end - skip) { + let i = skip + j; + this.checked_push_at(i, aggregate(&batches, j))?; + } + + Ok(()) + }, + ) + } + + pub fn compute_sum_of_others( + &mut self, + max_from: V::I, + others: &[&O], + exit: &Exit, + ) -> Result<()> + where + O: ReadableVec, + V::T: Add, + { + self.compute_aggregate_of_others(max_from, others, exit, |batches, j| { + batches + .iter() + .map(|b| b[j].clone()) + .reduce(|sum, v| sum + v) + .unwrap() + }) + } + + pub fn compute_min_of_others( + &mut self, + max_from: V::I, + others: &[&O], + exit: &Exit, + ) -> Result<()> + where + O: ReadableVec, + V::T: Add + Ord, + { + self.compute_aggregate_of_others(max_from, others, exit, |batches, j| { + batches.iter().map(|b| &b[j]).min().unwrap().clone() + }) + } + + pub fn compute_max_of_others( + &mut self, + max_from: V::I, + others: &[&O], + exit: &Exit, + ) -> Result<()> + where + O: ReadableVec, + V::T: Add + Ord, + { + self.compute_aggregate_of_others(max_from, others, exit, |batches, j| { + batches.iter().map(|b| &b[j]).max().unwrap().clone() + }) + } + + /// Computes weighted average: sum(weight_i * value_i) / sum(weight_i) + /// + /// Takes parallel slices of weight and value vecs from multiple sources. + /// For each index, computes the weighted average across all sources. + /// Returns zero if total weight is zero. + pub fn compute_weighted_average_of_others( + &mut self, + max_from: V::I, + weights: &[&OW], + values: &[&OV], + exit: &Exit, + ) -> Result<()> + where + W: VecValue + Into, + OW: ReadableVec, + OV: ReadableVec, + V::T: Into + From, + { + if weights.len() != values.len() { + return Err(Error::InvalidArgument( + "weights and values must have same length", + )); + } + + if weights.is_empty() { + return Err(Error::InvalidArgument( + "weights and values must have at least one element", + )); + } + + self.compute_init( + weights.iter().map(|v| v.version()).sum::() + + values.iter().map(|v| v.version()).sum(), + max_from, + exit, + |this| { + let skip = this.len(); + + let source_end = weights + .iter() + .map(|w| w.len()) + .chain(values.iter().map(|v| v.len())) + .min() + .unwrap_or(0); + let end = this.batch_end(source_end); + + if skip >= end { + return Ok(()); + } + + let weight_batches: Vec> = weights + .iter() + .map(|w| w.collect_range_at(skip, end)) + .collect(); + let value_batches: Vec> = values + .iter() + .map(|v| v.collect_range_at(skip, end)) + .collect(); + + for j in 0..(end - skip) { + let i = skip + j; + let mut total_weight = 0.0_f64; + let mut weighted_sum = 0.0_f64; + + for (w_batch, v_batch) in weight_batches.iter().zip(value_batches.iter()) { + let weight: f64 = w_batch[j].clone().into(); + let value: f64 = v_batch[j].clone().into(); + + if weight > 0.0 { + total_weight += weight; + weighted_sum += weight * value; + } + } + + let result = if total_weight > 0.0 { + V::T::from(weighted_sum / total_weight) + } else { + V::T::from(0.0) + }; + + this.checked_push_at(i, result)?; + } + + Ok(()) + }, + ) + } + + pub fn compute_sum_from_indexes( + &mut self, + max_from: V::I, + first_indexes: &impl ReadableVec, + indexes_count: &impl ReadableVec, + source: &(impl ReadableVec + Sized), + exit: &Exit, + ) -> Result<()> + where + V::T: Default + SaturatingAdd, + A: VecIndex + VecValue, + B: VecValue, + usize: From, + { + self.compute_init( + first_indexes.version() + indexes_count.version() + source.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = indexes_count.len(); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let pos = if skip < first_indexes.len() { + first_indexes.collect_one_at(skip).unwrap().to_usize() + } else { + return Ok(()); + }; + + let counts_batch: Vec = indexes_count + .collect_range_at(skip, end) + .into_iter() + .map(usize::from) + .collect(); + let total_count: usize = counts_batch.iter().sum(); + + let mut group_idx = 0usize; + + // Skip leading zero-count groups + while group_idx < counts_batch.len() && counts_batch[group_idx] == 0 { + this.push(V::T::default()); + group_idx += 1; + } + + if group_idx < counts_batch.len() { + let mut remaining = counts_batch[group_idx]; + + source.fold_range_at( + pos, + pos + total_count, + V::T::default(), + |sum, val: V::T| { + let sum = sum.saturating_add(val); + remaining -= 1; + if unlikely(remaining == 0) { + this.push(sum); + group_idx += 1; + while group_idx < counts_batch.len() && counts_batch[group_idx] == 0 + { + this.push(V::T::default()); + group_idx += 1; + } + if group_idx < counts_batch.len() { + remaining = counts_batch[group_idx]; + } + V::T::default() + } else { + sum + } + }, + ); + } + + Ok(()) + }, + ) + } + + pub fn compute_filtered_sum_from_indexes( + &mut self, + max_from: V::I, + first_indexes: &impl ReadableVec, + indexes_count: &impl ReadableVec, + source: &(impl ReadableVec + Sized), + mut filter: impl FnMut(&V::T) -> bool, + exit: &Exit, + ) -> Result<()> + where + V::T: Default + SaturatingAdd, + A: VecIndex + VecValue, + B: VecValue, + usize: From, + { + self.compute_init( + first_indexes.version() + indexes_count.version() + source.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = indexes_count.len(); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let pos = if skip < first_indexes.len() { + first_indexes.collect_one_at(skip).unwrap().to_usize() + } else { + return Ok(()); + }; + + let counts_batch: Vec = indexes_count + .collect_range_at(skip, end) + .into_iter() + .map(usize::from) + .collect(); + let total_count: usize = counts_batch.iter().sum(); + + let mut group_idx = 0usize; + + while group_idx < counts_batch.len() && counts_batch[group_idx] == 0 { + this.push(V::T::default()); + group_idx += 1; + } + + if group_idx < counts_batch.len() { + let mut remaining = counts_batch[group_idx]; + + source.fold_range_at( + pos, + pos + total_count, + V::T::default(), + |sum, val: V::T| { + let sum = if filter(&val) { + sum.saturating_add(val) + } else { + sum + }; + remaining -= 1; + if unlikely(remaining == 0) { + this.push(sum); + group_idx += 1; + while group_idx < counts_batch.len() && counts_batch[group_idx] == 0 + { + this.push(V::T::default()); + group_idx += 1; + } + if group_idx < counts_batch.len() { + remaining = counts_batch[group_idx]; + } + V::T::default() + } else { + sum + } + }, + ); + } + + Ok(()) + }, + ) + } + + pub fn compute_count_from_indexes( + &mut self, + max_from: V::I, + first_indexes: &impl ReadableVec, + other_to_else: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: From, + A: VecValue + + VecIndex + + Copy + + Add + + CheckedSub + + TryInto + + Default, + >::Error: core::error::Error + 'static, + B: VecValue, + { + self.compute_count_from_indexes_with( + max_from, + first_indexes, + other_to_else, + |first, next_first| next_first - first, + exit, + ) + } + + pub fn compute_filtered_count_from_indexes( + &mut self, + max_from: V::I, + first_indexes: &impl ReadableVec, + other_to_else: &impl ReadableVec, + mut filter: impl FnMut(A) -> bool, + exit: &Exit, + ) -> Result<()> + where + V::T: From, + A: VecValue + + VecIndex + + Copy + + Add + + CheckedSub + + TryInto + + Default, + B: VecValue, + >::Error: core::error::Error + 'static, + { + self.compute_count_from_indexes_with( + max_from, + first_indexes, + other_to_else, + |first, next_first| (first..next_first).filter(|i| filter(A::from(*i))).count(), + exit, + ) + } + + fn compute_count_from_indexes_with( + &mut self, + max_from: V::I, + first_indexes: &(impl ReadableVec + Sized), + other_to_else: &impl ReadableVec, + mut count_fn: impl FnMut(usize, usize) -> usize, + exit: &Exit, + ) -> Result<()> + where + V::T: From, + A: VecValue + + VecIndex + + Copy + + Add + + CheckedSub + + TryInto + + Default, + B: VecValue, + >::Error: core::error::Error + 'static, + { + self.compute_init( + first_indexes.version() + other_to_else.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = first_indexes.len(); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let prev_val = first_indexes.collect_one_at(skip).unwrap().to_usize(); + + let last_prev = + first_indexes.fold_range_at(skip + 1, end, prev_val, |prev, fi: A| { + let next = fi.to_usize(); + this.push(V::T::from(A::from(count_fn(prev, next)))); + next + }); + + let next_first = if end < first_indexes.len() { + first_indexes.collect_one_at(end).unwrap().to_usize() + } else { + other_to_else.len() + }; + this.push(V::T::from(A::from(count_fn(last_prev, next_first)))); + + Ok(()) + }, + ) + } +} diff --git a/crates/vecdb/src/variants/eager/compute/arithmetic.rs b/crates/vecdb/src/variants/eager/compute/arithmetic.rs new file mode 100644 index 000000000..d00b3887f --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/arithmetic.rs @@ -0,0 +1,176 @@ +use std::ops::{Add, Div, Mul, Sub}; + +use crate::{CheckedSub, Exit, ReadableVec, Result, StoredVec, VecValue}; + +use super::super::EagerVec; + +impl EagerVec +where + V: StoredVec, +{ + pub fn compute_add( + &mut self, + max_from: V::I, + added: &impl ReadableVec, + adder: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: Add, + { + self.compute_transform2( + max_from, + added, + adder, + |(i, v1, v2, ..)| (i, (v1 + v2)), + exit, + ) + } + + pub fn compute_subtract( + &mut self, + max_from: V::I, + subtracted: &impl ReadableVec, + subtracter: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: CheckedSub, + { + self.compute_transform2( + max_from, + subtracted, + subtracter, + |(i, v1, v2, ..)| { + ( + i, + v1.checked_sub(v2) + .expect("subtraction underflow in compute_subtract"), + ) + }, + exit, + ) + } + + pub fn compute_multiply( + &mut self, + max_from: V::I, + multiplied: &impl ReadableVec, + multiplier: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + V::T: From + Mul, + { + self.compute_transform2( + max_from, + multiplied, + multiplier, + |(i, v1, v2, ..)| (i, V::T::from(v1) * v2), + exit, + ) + } + + pub fn compute_divide( + &mut self, + max_from: V::I, + divided: &impl ReadableVec, + divider: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + V::T: From + Div, + { + self.compute_transform2( + max_from, + divided, + divider, + |(i, v1, v2, ..)| (i, V::T::from(v1) / v2), + exit, + ) + } + + pub fn compute_percentage( + &mut self, + max_from: V::I, + divided: &impl ReadableVec, + divider: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + V::T: From + + From + + From + + Mul + + Div + + Sub + + Copy, + { + self.compute_percentage_(max_from, divided, divider, exit, false) + } + + pub fn compute_percentage_difference( + &mut self, + max_from: V::I, + divided: &impl ReadableVec, + divider: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + V::T: From + + From + + From + + Mul + + Div + + Sub + + Copy, + { + self.compute_percentage_(max_from, divided, divider, exit, true) + } + + fn compute_percentage_( + &mut self, + max_from: V::I, + divided: &impl ReadableVec, + divider: &impl ReadableVec, + exit: &Exit, + as_difference: bool, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + V::T: From + + From + + From + + Mul + + Div + + Sub + + Copy, + { + let multiplier = V::T::from(100u8); + self.compute_transform2( + max_from, + divided, + divider, + move |(i, v1, v2, ..)| { + let divided = V::T::from(v1); + let divider = V::T::from(v2); + let v = divided * multiplier; + let mut v = v / divider; + if as_difference { + v = v - multiplier; + } + (i, v) + }, + exit, + ) + } +} diff --git a/crates/vecdb/src/variants/eager/compute/cumulative.rs b/crates/vecdb/src/variants/eager/compute/cumulative.rs new file mode 100644 index 000000000..d4e0d60e5 --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/cumulative.rs @@ -0,0 +1,242 @@ +use std::ops::{Add, AddAssign}; + +use crate::{AnyVec, Exit, ReadableVec, Result, StoredVec, VecIndex, VecValue, WritableVec}; + +use super::super::EagerVec; + +impl EagerVec +where + V: StoredVec, +{ + /// Compute cumulative sum from a source vec. + /// + /// Each value in the result is the sum of all values from the source up to + /// and including that index. + pub fn compute_cumulative( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + S: VecValue + Into, + V::T: Default + AddAssign + Copy, + { + self.compute_init(source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + let mut cumulative_val = if skip > 0 { + this.collect_one_at(skip - 1).unwrap() + } else { + V::T::default() + }; + + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), v: S| { + cumulative_val += v.into(); + this.checked_push_at(i, cumulative_val)?; + i += 1; + Ok(()) + }) + }) + } + + /// Compute cumulative sum from adding two source vecs element-wise. + /// + /// Each value in the result is the cumulative sum of `source1[i] + source2[i]` + /// for all indices up to and including i. + pub fn compute_cumulative_binary( + &mut self, + max_from: V::I, + source1: &impl ReadableVec, + source2: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + S1: VecValue + Into, + S2: VecValue + Into, + V::T: Default + AddAssign + Add + Copy, + { + self.compute_cumulative_transformed_binary( + max_from, + source1, + source2, + |v1: S1, v2: S2| v1.into() + v2.into(), + exit, + ) + } + + /// Compute cumulative sum from a custom binary transform of two source vecs. + /// + /// Each value in the result is the cumulative sum of `transform(source1[i], source2[i])` + /// for all indices up to and including i. + pub fn compute_cumulative_transformed_binary( + &mut self, + max_from: V::I, + source1: &impl ReadableVec, + source2: &impl ReadableVec, + mut transform: F, + exit: &Exit, + ) -> Result<()> + where + S1: VecValue, + S2: VecValue, + V::T: Default + AddAssign + Copy, + F: FnMut(S1, S2) -> V::T, + { + let target_len = source1.len().min(source2.len()); + + self.compute_init( + source1.version() + source2.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let end = this.batch_end(target_len); + if skip >= end { + return Ok(()); + } + + let mut cumulative_val = if skip > 0 { + this.collect_one_at(skip - 1).unwrap() + } else { + V::T::default() + }; + + let batch2 = source2.collect_range_at(skip, end); + let mut iter2 = batch2.into_iter(); + let mut i = skip; + + source1.try_fold_range_at(skip, end, (), |(), v1: S1| { + let v2 = iter2.next().unwrap(); + cumulative_val += transform(v1, v2); + this.checked_push_at(i, cumulative_val)?; + i += 1; + Ok(()) + }) + }, + ) + } + + /// Compute cumulative count of values matching a predicate. + /// + /// Each value in the result is the count of values from the source up to + /// and including that index where the predicate returns true. + pub fn compute_cumulative_count( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + predicate: P, + exit: &Exit, + ) -> Result<()> + where + S: VecValue, + V::T: From + AddAssign + Copy, + P: Fn(&S) -> bool, + { + self.compute_cumulative_count_from(max_from, source, V::I::from(0), predicate, exit) + } + + /// Compute rolling count of values matching a predicate within a window. + pub fn compute_rolling_count( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window_size: usize, + predicate: P, + exit: &Exit, + ) -> Result<()> + where + S: VecValue, + V::T: From + Into + Copy, + P: Fn(&S) -> bool, + { + self.compute_init(source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + // Recover count from stored output instead of rebuilding full window. + // Reads 1 element from self instead of window_size from source. + let mut count: usize = if skip > 0 { + this.collect_one_at(skip - 1).unwrap().into() + } else { + 0 + }; + + // Collect only the elements that leave the window during this batch. + // At position i, source[i - window_size] leaves (when i >= window_size). + // Reads batch_size elements instead of window_size. + let leave_start = skip.saturating_sub(window_size); + let leave_end = end.saturating_sub(window_size); + let leave_batch = if leave_end > leave_start { + source.collect_range_at(leave_start, leave_end) + } else { + vec![] + }; + + let mut leave_idx = 0; + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), v: S| { + if i >= window_size { + if predicate(&leave_batch[leave_idx]) { + count -= 1; + } + leave_idx += 1; + } + if predicate(&v) { + count += 1; + } + + this.checked_push_at(i, V::T::from(count))?; + i += 1; + Ok(()) + }) + }) + } + + /// Compute cumulative count of values matching a predicate, starting from a specific index. + /// + /// Values before `from` will be 0. Starting at `from`, counts values where predicate is true. + pub fn compute_cumulative_count_from( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + from: V::I, + predicate: P, + exit: &Exit, + ) -> Result<()> + where + S: VecValue, + V::T: From + AddAssign + Copy, + P: Fn(&S) -> bool, + { + let from_usize = from.to_usize(); + let mut count: Option = None; + self.compute_transform( + max_from, + source, + |(i, v, this)| { + let idx = i.to_usize(); + if count.is_none() { + count = Some(if idx > 0 { + this.collect_one_at(idx - 1).unwrap() + } else { + V::T::from(0_usize) + }); + } + if idx >= from_usize && predicate(&v) { + *count.as_mut().unwrap() += V::T::from(1_usize); + } + (i, count.unwrap()) + }, + exit, + ) + } +} diff --git a/crates/vecdb/src/variants/eager/compute/lookback.rs b/crates/vecdb/src/variants/eager/compute/lookback.rs new file mode 100644 index 000000000..1252c7cb7 --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/lookback.rs @@ -0,0 +1,374 @@ +use crate::{ + AnyVec, CheckedSub, Error, Exit, ReadableVec, Result, StoredVec, VecIndex, VecValue, + WritableVec, +}; + +use super::super::EagerVec; + +impl EagerVec +where + V: StoredVec, + V::I: CheckedSub, +{ + fn compute_with_lookback( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + lookback_len: usize, + exit: &Exit, + transform: F, + ) -> Result<()> + where + A: VecValue + Default, + F: Fn(usize, A, A) -> V::T, + { + self.compute_init(source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + // Collect only the values that will be looked back to during this batch. + // At position i, we need source[i - lookback_len] (when i >= lookback_len). + // Reads batch_size elements instead of lookback_len. + let prev_start = skip.saturating_sub(lookback_len); + let prev_end = end.saturating_sub(lookback_len); + let prev_batch = if prev_end > prev_start { + source.collect_range_at(prev_start, prev_end) + } else { + vec![] + }; + + let mut prev_idx = 0; + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), current: A| { + let previous = if i >= lookback_len { + let val = prev_batch[prev_idx].clone(); + prev_idx += 1; + val + } else { + A::default() + }; + let result = transform(i, current, previous); + this.checked_push_at(i, result)?; + i += 1; + Ok(()) + }) + }) + } + + pub fn compute_previous_value( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + len: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Default, + f32: From, + V::T: From, + { + self.compute_with_lookback(max_from, source, len, exit, |i, _, previous| { + if i < len { + V::T::from(f32::NAN) + } else { + V::T::from(f32::from(previous)) + } + }) + } + + /// Compute N-period change. Converts source values to output type before subtraction + /// to properly handle negative changes (e.g., unsigned source to signed output). + pub fn compute_change( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + len: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Default + Into, + V::T: CheckedSub + Default, + { + self.compute_with_lookback(max_from, source, len, exit, |i, current, previous| { + if i < len { + V::T::default() + } else { + let current: V::T = current.into(); + let previous: V::T = previous.into(); + current.checked_sub(previous).unwrap() + } + }) + } + + fn compute_ratio_change_scaled( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + len: usize, + multiplier: f32, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Default, + f32: From, + V::T: From, + { + self.compute_with_lookback(max_from, source, len, exit, |i, current, previous| { + if i < len { + V::T::from(f32::NAN) + } else { + let current_f32 = f32::from(current); + let previous_f32 = f32::from(previous); + V::T::from(((current_f32 / previous_f32) - 1.0) * multiplier) + } + }) + } + + pub fn compute_ratio_change( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + len: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Default, + f32: From, + V::T: From, + { + self.compute_ratio_change_scaled(max_from, source, len, 1.0, exit) + } + + pub fn compute_percentage_change( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + len: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Default, + f32: From, + V::T: From, + { + self.compute_ratio_change_scaled(max_from, source, len, 100.0, exit) + } + + /// Shared helper for rolling computations using variable window starts. + pub fn compute_rolling_from_window_starts( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + compute: F, + ) -> Result<()> + where + A: VecValue, + f64: From, + V::T: From, + F: Fn(f64, f64) -> f64, + { + self.compute_init( + window_starts.version() + values.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(values.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + let starts_batch = window_starts.collect_range_at(skip, end); + + // Pre-collect values from earliest ago height to end in one shot. + // Window starts are monotonically non-decreasing, so first is the minimum. + let min_start = starts_batch[0].to_usize().min(skip); + let values_data = values.collect_range_at(min_start, end); + + for (j, start) in starts_batch.into_iter().enumerate() { + let i = skip + j; + let start_usize = start.to_usize(); + let current = f64::from(values_data[i - min_start].clone()); + let result = if start_usize > i { + compute(current, f64::NAN) + } else if start_usize == i { + compute(current, current) + } else { + let previous = f64::from(values_data[start_usize - min_start].clone()); + compute(current, previous) + }; + this.checked_push_at(i, V::T::from(result))?; + } + + Ok(()) + }, + ) + } + + /// Compute ratio change using variable window starts (lookback vec). + /// For each index i, computes `values[i] / values[window_starts[i]] - 1`. + pub fn compute_rolling_ratio_change( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + f64: From, + V::T: From, + { + self.compute_rolling_from_window_starts( + max_from, + window_starts, + values, + exit, + |current, previous| { + if previous.is_nan() || previous == 0.0 { + f64::NAN + } else { + current / previous - 1.0 + } + }, + ) + } + + /// Compute percentage change using variable window starts (lookback vec). + /// For each index i, computes `(values[i] / values[window_starts[i]] - 1) * 100`. + pub fn compute_rolling_percentage_change( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + f64: From, + V::T: From, + { + self.compute_rolling_from_window_starts( + max_from, + window_starts, + values, + exit, + |current, previous| { + if previous.is_nan() || previous == 0.0 { + f64::NAN + } else { + (current / previous - 1.0) * 100.0 + } + }, + ) + } + + /// Compute change using variable window starts (lookback vec). + /// For each index i, computes `values[i] - values[window_starts[i]]`. + pub fn compute_rolling_change( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + f64: From, + V::T: From, + { + self.compute_rolling_from_window_starts( + max_from, + window_starts, + values, + exit, + |current, previous| { + if previous.is_nan() { + 0.0 + } else { + current - previous + } + }, + ) + } + + pub fn compute_cagr( + &mut self, + max_from: V::I, + percentage_returns: &impl ReadableVec, + days: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Default, + f32: From, + V::T: From, + { + if days == 0 || !days.is_multiple_of(365) { + return Err(Error::InvalidArgument( + "days must be non-zero and a multiple of 365", + )); + } + + let years = days / 365; + + self.compute_transform( + max_from, + percentage_returns, + |(i, percentage, ..)| { + let cagr = (((f32::from(percentage) / 100.0 + 1.0).powf(1.0 / years as f32)) - 1.0) + * 100.0; + (i, V::T::from(cagr)) + }, + exit, + ) + } + + /// For each index `i`, `output[i] = source[window_starts[i]]`. + /// Efficiently caches lookups since window_starts are monotonically non-decreasing. + pub fn compute_lookback( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + source: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + V::T: From, + { + self.compute_init( + window_starts.version() + source.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(source.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + let starts_batch = window_starts.collect_range_at(skip, end); + + // Pre-collect source from earliest ago height to end. + let min_start = starts_batch[0].to_usize().min(skip); + let source_data = source.collect_range_at(min_start, end); + + for (j, start) in starts_batch.into_iter().enumerate() { + let start_usize = start.to_usize(); + let value = source_data[start_usize - min_start].clone(); + this.checked_push_at(skip + j, V::T::from(value))?; + } + + Ok(()) + }, + ) + } +} diff --git a/crates/vecdb/src/variants/eager/compute/mod.rs b/crates/vecdb/src/variants/eager/compute/mod.rs new file mode 100644 index 000000000..fbf79e827 --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/mod.rs @@ -0,0 +1,6 @@ +mod aggregates; +mod arithmetic; +mod cumulative; +mod lookback; +mod statistics; +mod transforms; diff --git a/crates/vecdb/src/variants/eager/compute/statistics.rs b/crates/vecdb/src/variants/eager/compute/statistics.rs new file mode 100644 index 000000000..8b16badb2 --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/statistics.rs @@ -0,0 +1,1183 @@ +use std::{ + cmp::Ordering, + collections::VecDeque, + ops::{AddAssign, Div, Sub, SubAssign}, +}; + +use crate::{ + AnyVec, CheckedSub, Error, Exit, ReadableVec, Result, StoredVec, VecIndex, VecValue, Version, + WritableVec, +}; + +use super::super::EagerVec; + +impl EagerVec +where + V: StoredVec, +{ + fn compute_monotonic_window( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window: usize, + exit: &Exit, + should_pop: F, + ) -> Result<()> + where + A: VecValue + Ord, + V::T: From, + F: Fn(&A, &A) -> bool, + { + #[inline] + fn update_deque( + deque: &mut VecDeque<(usize, A)>, + i: usize, + value: A, + window: usize, + should_pop: &impl Fn(&A, &A) -> bool, + ) { + while let Some(&(idx, _)) = deque.front() { + if i >= window && idx <= i - window { + deque.pop_front(); + } else { + break; + } + } + while let Some((_, v)) = deque.back() { + if should_pop(v, &value) { + deque.pop_back(); + } else { + break; + } + } + deque.push_back((i, value)); + } + + self.compute_init(source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + let mut deque: VecDeque<(usize, A)> = VecDeque::with_capacity(window.min(1024)); + + // Rebuild deque state from source + let rebuild_start = skip.saturating_sub(window); + let mut rebuild_i = rebuild_start; + source.for_each_range_dyn_at(rebuild_start, skip, &mut |value: A| { + update_deque(&mut deque, rebuild_i, value, window, &should_pop); + rebuild_i += 1; + }); + + // Process new elements + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), value: A| { + update_deque(&mut deque, i, value, window, &should_pop); + + let v = deque.front().unwrap().1.clone(); + this.checked_push_at(i, V::T::from(v))?; + i += 1; + Ok(()) + }) + }) + } + + pub fn compute_max( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Ord, + V::T: From, + { + self.compute_monotonic_window(max_from, source, window, exit, |v, value| v < value) + } + + pub fn compute_min( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window: usize, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Ord, + V::T: From, + { + self.compute_monotonic_window(max_from, source, window, exit, |v, value| v > value) + } + + pub fn compute_sum( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window: usize, + exit: &Exit, + ) -> Result<()> + where + V::T: std::ops::Add + From + Default + CheckedSub, + A: VecValue, + { + // Cursor for the leaving-value reads — persists across batches so each + // compressed page is decompressed at most once instead of once per element. + let mut leaving = source.cursor(); + + self.compute_init(Version::new(2) + source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + let mut prev_sum = if skip > 0 { + this.collect_one_at(skip - 1).unwrap() + } else { + V::T::default() + }; + + // Position cursor at the start of the leaving-values window. + let pop_start = skip.saturating_sub(window); + if leaving.position() < pop_start { + leaving.advance(pop_start - leaving.position()); + } + + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), value: A| { + let value = V::T::from(value); + + let sum = if i >= window { + let old = V::T::from(leaving.next().unwrap()); + match prev_sum.clone().checked_sub(old) { + Some(diff) => diff + value, + None => return Err(Error::Underflow), + } + } else { + prev_sum.clone() + value + }; + + prev_sum = sum.clone(); + this.checked_push_at(i, sum)?; + i += 1; + Ok(()) + }) + }) + } + + /// Compute rolling sum with variable window starts. + /// For each index i, computes sum of values from `window_starts[i]` to i (inclusive). + pub fn compute_rolling_sum( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + V::T: From + Default + AddAssign + SubAssign, + { + // Cursor for the leaving-value reads — persists across batches so each + // compressed page is decompressed at most once instead of once per element. + let mut leaving = values.cursor(); + + self.compute_init( + window_starts.version() + values.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(values.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + let (mut running_sum, mut prev_start) = if skip > 0 { + let prev_idx = skip - 1; + let prev_start = window_starts.collect_one_at(prev_idx).unwrap(); + let sum = this.collect_one_at(prev_idx).unwrap(); + // Position cursor at the current window start. + if leaving.position() < prev_start.to_usize() { + leaving.advance(prev_start.to_usize() - leaving.position()); + } + (sum, prev_start) + } else { + (V::T::default(), V::I::from(0)) + }; + + let starts_batch = window_starts.collect_range_at(skip, end); + let values_batch = values.collect_range_at(skip, end); + + for (j, (start, value)) in starts_batch.into_iter().zip(values_batch).enumerate() { + let i = skip + j; + running_sum += V::T::from(value); + + if prev_start < start { + let n = start.to_usize() - prev_start.to_usize(); + leaving.for_each(n, |v: A| { + running_sum -= V::T::from(v); + }); + prev_start = start; + } + + this.checked_push_at(i, running_sum.clone())?; + } + + Ok(()) + }, + ) + } + + /// Compute rolling average with variable window starts. + /// For each index i, computes mean of values from `window_starts[i]` to i (inclusive). + pub fn compute_rolling_average( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + f64: From + From, + V::T: From + Default, + { + // Cursor for the leaving-value reads — persists across batches so each + // compressed page is decompressed at most once instead of once per element. + let mut leaving = values.cursor(); + + self.compute_init( + window_starts.version() + values.version() + Version::new(2), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(values.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + // Recover running_sum from stored average (fast but lossy). + let (mut running_sum, mut prev_start) = if skip > 0 { + let prev_idx = skip - 1; + let prev_start = window_starts.collect_one_at(prev_idx).unwrap(); + if leaving.position() < prev_start.to_usize() { + leaving.advance(prev_start.to_usize() - leaving.position()); + } + let stored_avg = f64::from(this.collect_one_at(prev_idx).unwrap()); + let window_count = prev_idx + 1 - prev_start.to_usize(); + (stored_avg * window_count as f64, prev_start) + } else { + (0.0_f64, V::I::from(0)) + }; + + let starts_batch = window_starts.collect_range_at(skip, end); + let values_batch = values.collect_range_at(skip, end); + + for (j, (start, value)) in starts_batch.into_iter().zip(values_batch).enumerate() { + let i = skip + j; + running_sum += f64::from(value); + + if prev_start < start { + let n = start.to_usize() - prev_start.to_usize(); + leaving.for_each(n, |v: A| { + running_sum -= f64::from(v); + }); + prev_start = start; + } + + let count = i - start.to_usize() + 1; + let avg = running_sum / count as f64; + this.checked_push_at(i, V::T::from(avg))?; + } + + Ok(()) + }, + ) + } + + /// Compute rolling standard deviation with variable window starts. + /// For each index `i`, computes SD of values from `window_starts[i]` to `i` (inclusive), + /// using the provided rolling mean. + /// `SD = sqrt(E[X²] - E[X]²)` where `E[X²]` is the rolling mean of squares + /// and `E[X]` is the rolling mean from the `mean` parameter. + pub fn compute_rolling_sd( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + mean: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + f64: From + From + From, + V::T: From, + { + let mut leaving = values.cursor(); + + self.compute_init( + window_starts.version() + values.version() + mean.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(values.len()).min(mean.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + let (mut running_sum_sq, mut prev_start) = if skip > 0 { + let prev_idx = skip - 1; + let prev_start = window_starts.collect_one_at(prev_idx).unwrap(); + let count = (prev_idx + 1 - prev_start.to_usize()) as f64; + let sd_val = f64::from(this.collect_one_at(prev_idx).unwrap()); + let mean_val = f64::from(mean.collect_one_at(prev_idx).unwrap()); + let sum_sq = (sd_val * sd_val + mean_val * mean_val) * count; + if leaving.position() < prev_start.to_usize() { + leaving.advance(prev_start.to_usize() - leaving.position()); + } + (sum_sq, prev_start) + } else { + (0.0f64, V::I::from(0)) + }; + + let starts_batch = window_starts.collect_range_at(skip, end); + let values_batch = values.collect_range_at(skip, end); + let mean_batch = mean.collect_range_at(skip, end); + + for (j, ((start, value), m)) in starts_batch + .into_iter() + .zip(values_batch) + .zip(mean_batch) + .enumerate() + { + let i = skip + j; + let val = f64::from(value); + running_sum_sq += val * val; + + if prev_start < start { + let n = start.to_usize() - prev_start.to_usize(); + leaving.for_each(n, |v: A| { + let old = f64::from(v); + running_sum_sq -= old * old; + }); + prev_start = start; + } + + let count = (i - start.to_usize() + 1) as f64; + let mean_val = f64::from(m); + let variance = (running_sum_sq / count - mean_val * mean_val).max(0.0); + this.checked_push_at(i, V::T::from(variance.sqrt()))?; + } + + Ok(()) + }, + ) + } + + /// Compute expanding (all-time) standard deviation. + /// For each index `i`, computes SD of all values from 0 to `i` (inclusive). + /// `SD = sqrt(E[X²] - E[X]²)`. + pub fn compute_expanding_sd( + &mut self, + max_from: V::I, + values: &impl ReadableVec, + mean: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + f64: From + From + From, + V::T: From, + { + self.compute_init(values.version() + mean.version(), max_from, exit, |this| { + let skip = this.len(); + let source_len = values.len().min(mean.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + let mut running_sum_sq = if skip > 0 { + let count = skip as f64; + let sd_val = f64::from(this.collect_one_at(skip - 1).unwrap()); + let mean_val = f64::from(mean.collect_one_at(skip - 1).unwrap()); + (sd_val * sd_val + mean_val * mean_val) * count + } else { + 0.0f64 + }; + + let values_batch = values.collect_range_at(skip, end); + let mean_batch = mean.collect_range_at(skip, end); + + for (j, (value, m)) in values_batch.into_iter().zip(mean_batch).enumerate() { + let i = skip + j; + let val = f64::from(value); + running_sum_sq += val * val; + + let count = (i + 1) as f64; + let mean_val = f64::from(m); + let variance = (running_sum_sq / count - mean_val * mean_val).max(0.0); + this.checked_push_at(i, V::T::from(variance.sqrt()))?; + } + + Ok(()) + }) + } + + /// Compute rolling EMA with variable window starts. + /// For each index `i`, computes an exponential moving average with + /// `α = 2/(span+1)` where `span = i - window_starts[i] + 1`. + pub fn compute_rolling_ema( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + f64: From + From, + V::T: From + Default, + { + self.compute_rolling_exponential(max_from, window_starts, values, exit, |span| { + 2.0 / (span + 1.0) + }) + } + + /// Compute rolling RMA (Wilder's smoothing) with variable window starts. + /// `α = 1/span` where `span = i - window_starts[i] + 1`. + pub fn compute_rolling_rma( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + f64: From + From, + V::T: From + Default, + { + self.compute_rolling_exponential(max_from, window_starts, values, exit, |span| 1.0 / span) + } + + fn compute_rolling_exponential( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + values: &impl ReadableVec, + exit: &Exit, + alpha_fn: F, + ) -> Result<()> + where + A: VecValue, + f64: From + From, + V::T: From + Default, + F: Fn(f64) -> f64, + { + self.compute_init( + Version::new(2) + window_starts.version() + values.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(values.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + let mut prev = if skip > 0 { + f64::from(this.collect_one_at(skip - 1).unwrap()) + } else { + 0.0_f64 + }; + + let starts_batch = window_starts.collect_range_at(skip, end); + let values_batch = values.collect_range_at(skip, end); + + for (j, (start, value)) in starts_batch.into_iter().zip(values_batch).enumerate() { + let i = skip + j; + let span = (i - start.to_usize() + 1) as f64; + let alpha = alpha_fn(span); + let value = f64::from(value); + prev = alpha * value + (1.0 - alpha) * prev; + this.checked_push_at(i, V::T::from(prev))?; + } + + Ok(()) + }, + ) + } + + /// Compute rolling maximum with variable window starts (deque-based). + pub fn compute_rolling_max_from_starts( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + source: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Ord, + V::T: From, + { + self.compute_rolling_monotonic_from_starts( + max_from, + window_starts, + source, + exit, + |back, new| *back <= *new, + ) + } + + /// Compute rolling minimum with variable window starts (deque-based). + pub fn compute_rolling_min_from_starts( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + source: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + Ord, + V::T: From, + { + self.compute_rolling_monotonic_from_starts( + max_from, + window_starts, + source, + exit, + |back, new| *back >= *new, + ) + } + + fn compute_rolling_monotonic_from_starts( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + source: &impl ReadableVec, + exit: &Exit, + should_pop: F, + ) -> Result<()> + where + A: VecValue + Ord, + V::T: From, + F: Fn(&A, &A) -> bool, + { + self.compute_init( + window_starts.version() + source.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts.len().min(source.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + // Rebuild deque from source values in the current window + let mut deque: VecDeque<(usize, A)> = VecDeque::new(); + if skip > 0 { + let window_start = window_starts.collect_one_at(skip - 1).unwrap().to_usize(); + let window_values = source.collect_range_at(window_start, skip); + for (k, v) in (window_start..skip).zip(window_values) { + while deque.back().is_some_and(|(_, dv)| should_pop(dv, &v)) { + deque.pop_back(); + } + deque.push_back((k, v)); + } + } + + let starts_batch = window_starts.collect_range_at(skip, end); + let values_batch = source.collect_range_at(skip, end); + + for (j, (start, value)) in starts_batch.into_iter().zip(values_batch).enumerate() { + let i = skip + j; + let ws = start.to_usize(); + + while let Some(&(idx, _)) = deque.front() { + if idx < ws { + deque.pop_front(); + } else { + break; + } + } + + while deque.back().is_some_and(|(_, v)| should_pop(v, &value)) { + deque.pop_back(); + } + deque.push_back((i, value)); + + this.checked_push_at(i, V::T::from(deque.front().unwrap().1.clone()))?; + } + + Ok(()) + }, + ) + } + + /// Compute rolling ratio with variable window starts. + /// For each index i, computes `sum(numerator[window_starts[i]..=i]) / denominator[i]`. + pub fn compute_rolling_ratio( + &mut self, + max_from: V::I, + window_starts: &impl ReadableVec, + numerator: &impl ReadableVec, + denominator: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + f64: From + From + From, + V::T: From, + { + // Cursor for the leaving-value reads — persists across batches so each + // compressed page is decompressed at most once instead of once per element. + let mut leaving = numerator.cursor(); + + self.compute_init( + window_starts.version() + numerator.version() + denominator.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_len = window_starts + .len() + .min(numerator.len()) + .min(denominator.len()); + let end = this.batch_end(source_len); + if skip >= end { + return Ok(()); + } + + // Recover running_sum from stored_ratio * denom[prev_idx]. + // Reads 3 values instead of window_count from numerator. + // Falls back to full scan when prev denom is 0. + let (mut running_sum, mut prev_start) = if skip > 0 { + let prev_idx = skip - 1; + let prev_start = window_starts.collect_one_at(prev_idx).unwrap(); + let prev_denom = f64::from(denominator.collect_one_at(prev_idx).unwrap()); + if prev_denom != 0.0 { + let stored_ratio = f64::from(this.collect_one_at(prev_idx).unwrap()); + if leaving.position() < prev_start.to_usize() { + leaving.advance(prev_start.to_usize() - leaving.position()); + } + (stored_ratio * prev_denom, prev_start) + } else { + let mut sum = 0.0_f64; + // Full scan fallback — position cursor at window start + // so the main loop can subtract leaving values correctly. + if leaving.position() < prev_start.to_usize() { + leaving.advance(prev_start.to_usize() - leaving.position()); + } + numerator.for_each_range_dyn_at( + prev_start.to_usize(), + prev_idx + 1, + &mut |v: A| { + sum += f64::from(v); + }, + ); + (sum, prev_start) + } + } else { + (0.0_f64, V::I::from(0)) + }; + + let starts_batch = window_starts.collect_range_at(skip, end); + let num_batch = numerator.collect_range_at(skip, end); + let denom_batch = denominator.collect_range_at(skip, end); + + for (j, ((start, num_val), denom_val)) in starts_batch + .into_iter() + .zip(num_batch) + .zip(denom_batch) + .enumerate() + { + let i = skip + j; + running_sum += f64::from(num_val); + + if prev_start < start { + let n = start.to_usize() - prev_start.to_usize(); + leaving.for_each(n, |v: A| { + running_sum -= f64::from(v); + }); + prev_start = start; + } + + let denom = f64::from(denom_val); + let ratio = if denom != 0.0 { + running_sum / denom + } else { + 0.0 + }; + this.checked_push_at(i, V::T::from(ratio))?; + } + + Ok(()) + }, + ) + } + + pub fn compute_sma( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + sma: usize, + exit: &Exit, + ) -> Result<()> + where + V::T: std::ops::Add + From + From, + A: VecValue, + f32: From + From, + { + self.compute_sma_(max_from, source, sma, exit, None) + } + + pub fn compute_sma_( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window: usize, + exit: &Exit, + min_i: Option, + ) -> Result<()> + where + V::T: std::ops::Add + From + From, + A: VecValue, + f32: From + From, + { + self.compute_init(Version::new(2) + source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + let min_i = min_i.map(|i| i.to_usize()); + let min_prev_i = min_i.unwrap_or_default(); + + let mut prev_sma = if skip > 0 && skip > min_prev_i { + f32::from(this.collect_one_at(skip - 1).unwrap()) + } else { + 0.0 + }; + + // Collect only the values that leave the window during this batch. + // At position i, source[i - window] leaves (when i >= min_prev_i + window). + // Reads batch_size elements instead of window. + let pop_start = skip.saturating_sub(window).max(min_prev_i); + let pop_end = end.saturating_sub(window).max(pop_start); + let pop_batch: Vec = if pop_end > pop_start { + let mut v = Vec::with_capacity(pop_end - pop_start); + source.for_each_range_dyn_at(pop_start, pop_end, &mut |val: A| { + v.push(f32::from(val)); + }); + v + } else { + vec![] + }; + + let mut pop_idx = 0; + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), value: A| { + if min_i.is_none_or(|m| m <= i) { + let value_f32 = f32::from(value); + let effective_i = i - min_prev_i; + + let sma_result = if effective_i >= window { + let old = pop_batch[pop_idx]; + pop_idx += 1; + prev_sma + (value_f32 - old) / window as f32 + } else { + (prev_sma * effective_i as f32 + value_f32) / (effective_i + 1) as f32 + }; + + prev_sma = sma_result; + this.checked_push_at(i, V::T::from(sma_result))?; + } else { + this.checked_push_at(i, V::T::from(f32::NAN))?; + } + i += 1; + Ok(()) + }) + }) + } + + pub fn compute_rolling_median( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + window: usize, + exit: &Exit, + ) -> Result<()> + where + V::T: From, + A: VecValue, + f32: From, + { + #[inline] + fn median(buf: &VecDeque, scratch: &mut Vec) -> f32 { + scratch.clear(); + scratch.extend(buf.iter().copied()); + let cmp = |a: &f32, b: &f32| a.partial_cmp(b).unwrap_or(Ordering::Equal); + let mid = scratch.len() / 2; + scratch.select_nth_unstable_by(mid, cmp); + if scratch.len().is_multiple_of(2) { + let upper = scratch[mid]; + let lower = scratch[..mid] + .iter() + .copied() + .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)) + .unwrap_or(upper); + (lower + upper) / 2.0 + } else { + scratch[mid] + } + } + + self.compute_init(Version::new(2) + source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + // Median inherently needs the full window for sorting — rebuild it. + let mut buf: VecDeque = + VecDeque::with_capacity(window.saturating_add(1).min(1024)); + if skip > 0 { + let start = skip.saturating_sub(window); + source.for_each_range_dyn_at(start, skip, &mut |v: A| { + buf.push_back(f32::from(v)); + }); + } + + let mut scratch = Vec::with_capacity(window.min(1024)); + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), value: A| { + buf.push_back(f32::from(value)); + if buf.len() > window { + buf.pop_front(); + } + this.checked_push_at(i, V::T::from(median(&buf, &mut scratch)))?; + i += 1; + Ok(()) + }) + }) + } + + pub fn compute_ema( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + ema: usize, + exit: &Exit, + ) -> Result<()> + where + V::T: From + From, + A: VecValue, + f32: From + From, + { + self.compute_ema_(max_from, source, ema, exit, None) + } + + pub fn compute_ema_( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + ema: usize, + exit: &Exit, + min_i: Option, + ) -> Result<()> + where + V::T: From + From, + A: VecValue, + f32: From + From, + { + let k = 2.0 / (ema as f32 + 1.0); + self.compute_exponential_smoothing(max_from, source, ema, Version::new(3), k, min_i, exit) + } + + /// Compute Wilder's Running Moving Average (RMA). + /// Uses alpha = 1/period instead of EMA's 2/(period+1). + /// This is the standard smoothing method for RSI. + pub fn compute_rma( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + period: usize, + exit: &Exit, + ) -> Result<()> + where + V::T: From + From, + A: VecValue, + f32: From + From, + { + let k = 1.0 / period as f32; + self.compute_exponential_smoothing(max_from, source, period, Version::new(4), k, None, exit) + } + + /// Shared implementation for EMA and RMA. + /// - EMA: k = 2/(period+1) + /// - RMA (Wilder's): k = 1/period + #[allow(clippy::too_many_arguments)] + fn compute_exponential_smoothing( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + period: usize, + version: Version, + k: f32, + min_i: Option, + exit: &Exit, + ) -> Result<()> + where + V::T: From + From, + A: VecValue, + f32: From + From, + { + let one_minus_k = 1.0 - k; + + self.compute_init(version + source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + let min_start = min_i.map(|i| i.to_usize()).unwrap_or(0); + + let mut prev = if skip > 0 && skip > min_start { + this.collect_one_at(skip - 1).unwrap() + } else { + V::T::from(0.0) + }; + + let mut index = skip; + source.try_fold_range_at(skip, end, (), |(), value: A| { + if index >= min_start { + let processed = index - min_start + 1; + let value = f32::from(value); + + let p = f32::from(prev.clone()); + let result = if processed > period { + let p = if p.is_nan() { 0.0 } else { p }; + V::T::from(value * k + p * one_minus_k) + } else { + V::T::from((p * (processed - 1) as f32 + value) / processed as f32) + }; + + prev = result.clone(); + this.checked_push_at(index, result)?; + } else { + this.checked_push_at(index, V::T::from(f32::NAN))?; + } + + index += 1; + Ok(()) + }) + }) + } + + fn compute_all_time_extreme( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + exit: &Exit, + compare: F, + exclude_default: bool, + ) -> Result<()> + where + V::T: From + Ord + Default, + A: VecValue, + F: Fn(V::T, V::T) -> V::T + Copy, + { + let mut prev = None; + self.compute_transform( + max_from, + source, + |(i, v, this)| { + let v = V::T::from(v); + if prev.is_none() { + let idx = i.to_usize(); + prev = Some(if idx > 0 { + this.collect_one_at(idx - 1).unwrap() + } else { + v.clone() + }); + } + let extreme = compare(prev.as_ref().unwrap().clone(), v.clone()); + + prev.replace(if !exclude_default || extreme != V::T::default() { + extreme.clone() + } else { + // Keep the non-default value for future comparisons + if v != V::T::default() { + v + } else { + prev.as_ref().unwrap().clone() + } + }); + (i, extreme) + }, + exit, + ) + } + + /// Computes the all time high of a source. + /// This version is more optimized than `compute_max` with a window set to `usize::MAX`. + pub fn compute_all_time_high( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: From + Ord + Default, + A: VecValue, + { + self.compute_all_time_extreme(max_from, source, exit, |prev, v| prev.max(v), false) + } + + /// Computes the all time low of a source. + /// This version is more optimized than `compute_min` with a window set to `usize::MAX`. + pub fn compute_all_time_low( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: From + Ord + Default, + A: VecValue, + { + self.compute_all_time_low_(max_from, source, exit, false) + } + + /// Computes the all time low of a source. + /// This version is more optimized than `compute_min` with a window set to `usize::MAX`. + pub fn compute_all_time_low_( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + exit: &Exit, + exclude_default: bool, + ) -> Result<()> + where + V::T: From + Ord + Default, + A: VecValue, + { + self.compute_all_time_extreme( + max_from, + source, + exit, + |prev, v| prev.min(v), + exclude_default, + ) + } + + /// Computes the all time high starting from a specific index. + /// Values before `from` will be the default value (typically 0). + pub fn compute_all_time_high_from( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + from: V::I, + exit: &Exit, + ) -> Result<()> + where + V::T: From + Ord + Default + Copy, + A: VecValue, + { + self.compute_all_time_extreme_from(max_from, source, from, exit, V::T::max) + } + + /// Computes the all time low starting from a specific index. + /// Values before `from` will be the default value (typically 0). + pub fn compute_all_time_low_from( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + from: V::I, + exit: &Exit, + ) -> Result<()> + where + V::T: From + Ord + Default + Copy, + A: VecValue, + { + self.compute_all_time_extreme_from(max_from, source, from, exit, V::T::min) + } + + fn compute_all_time_extreme_from( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + from: V::I, + exit: &Exit, + compare: fn(V::T, V::T) -> V::T, + ) -> Result<()> + where + V::T: From + Ord + Default + Copy, + A: VecValue, + { + let from_usize = from.to_usize(); + let mut prev: Option = None; + self.compute_transform( + max_from, + source, + |(i, v, this)| { + let idx = i.to_usize(); + if prev.is_none() { + prev = Some(if idx > 0 { + this.collect_one_at(idx - 1).unwrap() + } else { + V::T::default() + }); + } + if idx >= from_usize { + *prev.as_mut().unwrap() = compare(prev.unwrap(), V::T::from(v)); + } + (i, prev.unwrap()) + }, + exit, + ) + } + + pub fn compute_zscore( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + sma: &impl ReadableVec, + sd: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: From, + A: VecValue + Sub + Div, + B: VecValue, + C: VecValue, + f32: From + From + From, + { + self.compute_transform3( + max_from, + source, + sma, + sd, + |(i, ratio, sma, sd, ..)| (i, (ratio - sma) / sd), + exit, + ) + } +} diff --git a/crates/vecdb/src/variants/eager/compute/transforms.rs b/crates/vecdb/src/variants/eager/compute/transforms.rs new file mode 100644 index 000000000..153417ea3 --- /dev/null +++ b/crates/vecdb/src/variants/eager/compute/transforms.rs @@ -0,0 +1,372 @@ +use crate::{ + AnyVec, BinaryTransform, Cursor, Exit, ReadableVec, Result, StoredVec, VecIndex, VecValue, + Version, WritableVec, +}; + +use super::super::EagerVec; + +impl EagerVec +where + V: StoredVec, +{ + pub fn compute_to( + &mut self, + max_from: V::I, + to: usize, + version: Version, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + F: FnMut(V::I) -> (V::I, V::T), + { + self.compute_init(version, max_from, exit, |this| { + let from = this.len(); + let end = this.batch_end(to); + if from >= end { + return Ok(()); + } + + for i in from..end { + let (idx, val) = t(V::I::from(i)); + this.checked_push(idx, val)?; + } + + Ok(()) + }) + } + + pub fn compute_range( + &mut self, + max_from: V::I, + other: &impl ReadableVec, + t: F, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + F: FnMut(V::I) -> (V::I, V::T), + { + self.compute_to(max_from, other.len(), other.version(), t, exit) + } + + pub fn compute_from_index( + &mut self, + max_from: V::I, + other: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::T: From, + A: VecValue, + { + self.compute_to( + max_from, + other.len(), + other.version(), + |i| (i, V::T::from(i)), + exit, + ) + } + + pub fn compute_transform( + &mut self, + max_from: V::I, + source: &impl ReadableVec, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + F: FnMut((V::I, A, &Self)) -> (V::I, V::T), + { + self.compute_init(source.version(), max_from, exit, |this| { + let skip = this.len(); + let end = this.batch_end(source.len()); + if skip >= end { + return Ok(()); + } + + let mut i = skip; + source.try_fold_range_at(skip, end, (), |(), b: A| { + let (idx, v) = t((V::I::from(i), b, &*this)); + i += 1; + this.checked_push(idx, v) + }) + }) + } + + pub fn compute_transform2( + &mut self, + max_from: V::I, + other1: &impl ReadableVec, + other2: &impl ReadableVec, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + F: FnMut((V::I, A, B, &Self)) -> (V::I, V::T), + { + self.compute_init( + other1.version() + other2.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = other1.len().min(other2.len()); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let batch2 = other2.collect_range_at(skip, end); + let mut iter2 = batch2.into_iter(); + let mut i = skip; + + other1.try_fold_range_at(skip, end, (), |(), b: A| { + let (idx, v) = t((V::I::from(i), b, iter2.next().unwrap(), &*this)); + i += 1; + this.checked_push(idx, v) + }) + }, + ) + } + + pub fn compute_binary( + &mut self, + max_from: V::I, + source1: &impl ReadableVec, + source2: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + F: BinaryTransform, + { + self.compute_transform2( + max_from, + source1, + source2, + |(h, a, b, ..)| (h, F::apply(a, b)), + exit, + ) + } + + pub fn compute_transform3( + &mut self, + max_from: V::I, + other1: &impl ReadableVec, + other2: &impl ReadableVec, + other3: &impl ReadableVec, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + C: VecValue, + F: FnMut((V::I, A, B, C, &Self)) -> (V::I, V::T), + { + self.compute_init( + other1.version() + other2.version() + other3.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = other1.len().min(other2.len()).min(other3.len()); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let batch2 = other2.collect_range_at(skip, end); + let batch3 = other3.collect_range_at(skip, end); + let mut iter2 = batch2.into_iter(); + let mut iter3 = batch3.into_iter(); + let mut i = skip; + + other1.try_fold_range_at(skip, end, (), |(), b: A| { + let (idx, v) = t(( + V::I::from(i), + b, + iter2.next().unwrap(), + iter3.next().unwrap(), + &*this, + )); + i += 1; + this.checked_push(idx, v) + }) + }, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn compute_transform4( + &mut self, + max_from: V::I, + other1: &impl ReadableVec, + other2: &impl ReadableVec, + other3: &impl ReadableVec, + other4: &impl ReadableVec, + mut t: F, + exit: &Exit, + ) -> Result<()> + where + A: VecValue, + B: VecValue, + C: VecValue, + D: VecValue, + F: FnMut((V::I, A, B, C, D, &Self)) -> (V::I, V::T), + { + self.compute_init( + other1.version() + other2.version() + other3.version() + other4.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let source_end = other1 + .len() + .min(other2.len()) + .min(other3.len()) + .min(other4.len()); + let end = this.batch_end(source_end); + if skip >= end { + return Ok(()); + } + + let batch2 = other2.collect_range_at(skip, end); + let batch3 = other3.collect_range_at(skip, end); + let batch4 = other4.collect_range_at(skip, end); + let mut iter2 = batch2.into_iter(); + let mut iter3 = batch3.into_iter(); + let mut iter4 = batch4.into_iter(); + let mut i = skip; + + other1.try_fold_range_at(skip, end, (), |(), b: A| { + let (idx, v) = t(( + V::I::from(i), + b, + iter2.next().unwrap(), + iter3.next().unwrap(), + iter4.next().unwrap(), + &*this, + )); + i += 1; + this.checked_push(idx, v) + }) + }, + ) + } + + /// Compute values through an indirection: for each index i, produces + /// `source2[source1[i]]`. Keys from source1 must be monotonically increasing + /// so that source2 access is sequential (cursor-friendly). + pub fn compute_indirect_sequential( + &mut self, + max_from: V::I, + source1: &impl ReadableVec, + source2: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + A: VecValue + VecIndex, + { + // Cursor persists across batches to avoid re-decompressing pages. + let mut cursor = Cursor::new(source2); + let mut cursor_pos: usize = 0; + let mut last_v: Option = None; + + self.compute_init( + source1.version() + source2.version(), + max_from, + exit, + |this| { + let skip = this.len(); + let end = this.batch_end(source1.len()); + if skip >= end { + return Ok(()); + } + + let keys: Vec = source1.collect_range_at(skip, end); + + for (j, key) in keys.into_iter().enumerate() { + let key_pos = key.to_usize(); + let v = if key_pos >= cursor_pos { + if key_pos > cursor_pos { + cursor.advance(key_pos - cursor_pos); + } + let v = cursor.next().unwrap(); + cursor_pos = key_pos + 1; + v + } else { + // Duplicate key from gap-filled periods — reuse previous value + last_v.clone().unwrap() + }; + last_v = Some(v.clone()); + let idx = V::I::from(skip + j); + this.checked_push(idx, v)?; + } + + Ok(()) + }, + ) + } + + pub fn compute_first_per_index( + &mut self, + max_from: V::T, + other: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> + where + V::I: VecValue + VecIndex, + V::T: VecIndex, + { + self.validate_computed_version_or_reset(other.version())?; + + self.repeat_until_complete(exit, |this| { + let skip = if this.len() > 0 { + this.collect_last() + .unwrap() + .to_usize() + .min(max_from.to_usize()) + } else { + 0 + }; + + let end = this.batch_end(other.len()); + if skip >= end { + return Ok(()); + } + + let mut prev_i = None; + let batch = other.collect_range_at(skip, end); + + if let Some(&first_target) = batch.first() { + this.truncate_if_needed(first_target)?; + } + + for (j, i) in batch.into_iter().enumerate() { + let v = V::T::from(skip + j); + debug_assert!(prev_i.is_none_or(|prev| prev <= i)); + if prev_i.is_some_and(|prev_i| prev_i == i) { + continue; + } + if this.collect_one(i).is_none_or(|old_v| old_v > v) { + // Pad gaps with the current value so empty periods get zero-length ranges + let i_usize = i.to_usize(); + while this.len() < i_usize { + this.push(v); + } + this.push(v); + } + prev_i.replace(i); + } + + Ok(()) + }) + } +} diff --git a/crates/vecdb/src/variants/eager/importable.rs b/crates/vecdb/src/variants/eager/importable.rs new file mode 100644 index 000000000..177f6922c --- /dev/null +++ b/crates/vecdb/src/variants/eager/importable.rs @@ -0,0 +1,23 @@ +use rawdb::Database; + +use crate::{ImportOptions, ImportableVec, Result, Version}; + +use super::EagerVec; + +impl ImportableVec for EagerVec { + fn import(db: &Database, name: &str, version: Version) -> Result { + Ok(Self(V::import(db, name, version)?)) + } + + fn import_with(options: ImportOptions) -> Result { + Ok(Self(V::import_with(options)?)) + } + + fn forced_import(db: &Database, name: &str, version: Version) -> Result { + Ok(Self(V::forced_import(db, name, version)?)) + } + + fn forced_import_with(options: ImportOptions) -> Result { + Ok(Self(V::forced_import_with(options)?)) + } +} diff --git a/crates/vecdb/src/variants/eager/mod.rs b/crates/vecdb/src/variants/eager/mod.rs new file mode 100644 index 000000000..e5134851e --- /dev/null +++ b/crates/vecdb/src/variants/eager/mod.rs @@ -0,0 +1,90 @@ +use log::info; + +mod any_stored_vec; +mod any_vec; +mod compute; +mod importable; +mod readable; +mod readable_cloneable; +mod stored; +mod typed; +mod writable; + +use crate::{ + AnyStoredVec, AnyVec, Exit, Result, StoredVec, Version, WritableVec, + traits::writable::MAX_CACHE_SIZE, +}; + +/// Wrapper for computing and storing derived values from source vectors. +/// +/// `EagerVec` wraps any `StoredVec` and provides computation methods to derive and persist +/// calculated values. Results are stored on disk and automatically recomputed when: +/// - Source data versions change +/// - The vector's computation logic version changes +/// +/// # Key Features +/// - **Incremental Updates**: Only computes missing values, not the entire dataset +/// - **Automatic Versioning**: Detects stale data and recomputes automatically +/// - **Batched Writes**: Flushes periodically to prevent excessive memory usage +/// +/// # Common Operations +/// - Transformations: `compute_transform()`, `compute_range()` +/// - Arithmetic: `compute_add()`, `compute_subtract()`, `compute_multiply()`, `compute_divide()` +/// - Moving statistics: `compute_sma()`, `compute_ema()`, `compute_sum()`, `compute_max()`, `compute_min()` +/// - Lookback calculations: `compute_change()`, `compute_percentage_change()` +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct EagerVec(pub(super) V); + +impl EagerVec +where + V: StoredVec, +{ + /// Validates version, truncates to `max_from`, then runs `f` in batched writes. + fn compute_init(&mut self, version: Version, max_from: V::I, exit: &Exit, f: F) -> Result<()> + where + F: FnMut(&mut Self) -> Result<()>, + { + self.validate_computed_version_or_reset(version)?; + self.truncate_if_needed(max_from)?; + self.repeat_until_complete(exit, f) + } + + /// Max end index for one batch, capped at `max_end`. + /// Ensures `pushed_len * SIZE_OF_T >= MAX_CACHE_SIZE` so `batch_limit_reached()` fires. + #[inline] + pub fn batch_end(&self, max_end: usize) -> usize { + let size = size_of::().max(1); + let cap = MAX_CACHE_SIZE.div_ceil(size); + (self.len() + cap).min(max_end) + } + + /// Helper that repeatedly calls a compute function until it completes. + /// Writes between iterations when batch limit is hit. + pub fn repeat_until_complete(&mut self, exit: &Exit, mut f: F) -> Result<()> + where + F: FnMut(&mut Self) -> Result<()>, + { + loop { + f(self)?; + let batch_limit_reached = self.batch_limit_reached(); + if batch_limit_reached { + info!("Batch limit reached, saving to disk..."); + } + if self.is_dirty() { + let _lock = exit.lock(); + self.write()?; + } + if !batch_limit_reached { + break; + } + } + + Ok(()) + } + + /// Removes this vector and all its associated regions from the database + pub fn remove(self) -> Result<()> { + self.0.remove() + } +} diff --git a/crates/vecdb/src/variants/eager/readable.rs b/crates/vecdb/src/variants/eager/readable.rs new file mode 100644 index 000000000..e216481c8 --- /dev/null +++ b/crates/vecdb/src/variants/eager/readable.rs @@ -0,0 +1,45 @@ +use crate::{ReadableVec, StoredVec}; + +use super::EagerVec; + +impl ReadableVec for EagerVec +where + V: StoredVec, +{ + #[inline(always)] + fn collect_one_at(&self, index: usize) -> Option { + self.0.collect_one_at(index) + } + + #[inline(always)] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + self.0.read_into_at(from, to, buf) + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(V::T)) { + self.0.for_each_range_dyn_at(from, to, f) + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, f: F) -> B + where + Self: Sized, + { + self.0.fold_range_at(from, to, init, f) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized, + { + self.0.try_fold_range_at(from, to, init, f) + } +} diff --git a/crates/vecdb/src/variants/eager/readable_cloneable.rs b/crates/vecdb/src/variants/eager/readable_cloneable.rs new file mode 100644 index 000000000..ec783e556 --- /dev/null +++ b/crates/vecdb/src/variants/eager/readable_cloneable.rs @@ -0,0 +1,13 @@ +use crate::{ReadableBoxedVec, ReadableCloneableVec, StoredVec}; + +use super::EagerVec; + +impl ReadableCloneableVec for EagerVec +where + V: StoredVec, +{ + #[inline] + fn read_only_boxed_clone(&self) -> ReadableBoxedVec { + self.0.read_only_boxed_clone() + } +} diff --git a/crates/vecdb/src/variants/eager/stored.rs b/crates/vecdb/src/variants/eager/stored.rs new file mode 100644 index 000000000..e229610b2 --- /dev/null +++ b/crates/vecdb/src/variants/eager/stored.rs @@ -0,0 +1,15 @@ +use crate::StoredVec; + +use super::EagerVec; + +impl StoredVec for EagerVec +where + V: StoredVec, +{ + type ReadOnly = V::ReadOnly; + + #[inline] + fn read_only_clone(&self) -> Self::ReadOnly { + self.0.read_only_clone() + } +} diff --git a/crates/vecdb/src/variants/eager/typed.rs b/crates/vecdb/src/variants/eager/typed.rs new file mode 100644 index 000000000..597bfac02 --- /dev/null +++ b/crates/vecdb/src/variants/eager/typed.rs @@ -0,0 +1,11 @@ +use crate::{StoredVec, TypedVec}; + +use super::EagerVec; + +impl TypedVec for EagerVec +where + V: StoredVec, +{ + type I = V::I; + type T = V::T; +} diff --git a/crates/vecdb/src/variants/eager/writable.rs b/crates/vecdb/src/variants/eager/writable.rs new file mode 100644 index 000000000..46495829f --- /dev/null +++ b/crates/vecdb/src/variants/eager/writable.rs @@ -0,0 +1,58 @@ +use std::{collections::BTreeMap, path::PathBuf}; + +use crate::{Result, Stamp, StoredVec, WritableVec}; + +use super::EagerVec; + +impl WritableVec for EagerVec +where + V: StoredVec, +{ + #[inline] + fn push(&mut self, value: V::T) { + self.0.push(value); + } + + #[inline] + fn pushed(&self) -> &[V::T] { + self.0.pushed() + } + + #[inline] + fn truncate_if_needed_at(&mut self, index: usize) -> Result<()> { + self.0.truncate_if_needed_at(index) + } + + #[inline] + fn reset(&mut self) -> Result<()> { + self.0.reset() + } + + #[inline] + fn reset_unsaved(&mut self) { + self.0.reset_unsaved() + } + + #[inline] + fn is_dirty(&self) -> bool { + self.0.is_dirty() + } + + #[inline] + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + self.0.stamped_write_with_changes(stamp) + } + + #[inline] + fn rollback(&mut self) -> Result<()> { + self.0.rollback() + } + + fn find_rollback_files(&self) -> Result> { + self.0.find_rollback_files() + } + + fn save_rollback_state(&mut self) { + self.0.save_rollback_state() + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/any_vec.rs b/crates/vecdb/src/variants/lazy/agg/any_vec.rs new file mode 100644 index 000000000..5e1c72d2c --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/any_vec.rs @@ -0,0 +1,44 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::LazyAggVec; + +impl AnyVec for LazyAggVec +where + I: VecIndex, + O: VecValue, + S1I: VecIndex, + S2T: VecValue, + S1T: VecValue, + Strat: 'static, +{ + fn version(&self) -> Version { + self.version + self.source.version() + self.mapping_version + } + + fn name(&self) -> &str { + &self.name + } + + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + fn len(&self) -> usize { + (self.mapping)().len() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + vec![] + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/clone.rs b/crates/vecdb/src/variants/lazy/agg/clone.rs new file mode 100644 index 000000000..eb356a2b8 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/clone.rs @@ -0,0 +1,25 @@ +use std::marker::PhantomData; + +use crate::{VecIndex, VecValue}; + +use super::LazyAggVec; + +impl Clone for LazyAggVec +where + I: VecIndex, + O: VecValue, + S1I: VecIndex, + S2T: VecValue, + S1T: VecValue, +{ + fn clone(&self) -> Self { + Self { + name: self.name.clone(), + version: self.version, + mapping_version: self.mapping_version, + source: self.source.clone(), + mapping: self.mapping.clone(), + _phantom: PhantomData, + } + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/fold.rs b/crates/vecdb/src/variants/lazy/agg/fold.rs new file mode 100644 index 000000000..7d1c0c01b --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/fold.rs @@ -0,0 +1,46 @@ +use crate::{ReadableVec, VecIndex, VecValue}; + +/// Aggregation strategy for [`super::LazyAggVec`]. +/// +/// Determines how values are produced from a source vec and a pre-materialized mapping. +/// Implement this on a zero-sized marker type to define a custom strategy. +/// +/// Built-in strategy: `Sparse`. +pub trait AggFold: 'static { + fn try_fold + ?Sized, B, E, F: FnMut(B, O) -> Result>( + source: &S, + mapping: &[S2T], + from: usize, + to: usize, + init: B, + f: F, + ) -> Result; + + fn fold + ?Sized, B, F: FnMut(B, O) -> B>( + source: &S, + mapping: &[S2T], + from: usize, + to: usize, + init: B, + mut f: F, + ) -> B { + match Self::try_fold(source, mapping, from, to, init, |b, o| { + Ok::<_, std::convert::Infallible>(f(b, o)) + }) { + Ok(b) => b, + Err(e) => match e {}, + } + } + + fn collect_one + ?Sized>( + source: &S, + mapping: &[S2T], + index: usize, + ) -> Option { + let mut result = None; + Self::fold(source, mapping, index, index + 1, (), |(), v| { + result = Some(v) + }); + result + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/mod.rs b/crates/vecdb/src/variants/lazy/agg/mod.rs new file mode 100644 index 000000000..8b70cd46a --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/mod.rs @@ -0,0 +1,61 @@ +use std::{marker::PhantomData, sync::Arc}; + +mod any_vec; +mod clone; +mod fold; +mod readable; +mod sparse; +mod typed; + +pub use fold::*; +use sparse::Sparse; + +use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; + +/// Lazy aggregation vector that maps coarser output indices to ranges in a finer source. +/// +/// Values are computed on-the-fly using cursor-based sequential access. +/// The mapping is pulled via a caller-provided closure on each read. +pub struct LazyAggVec +where + I: VecIndex, + O: VecValue, + S1I: VecIndex, + S2T: VecValue, + S1T: VecValue, +{ + pub(super) name: Arc, + pub(super) version: Version, + pub(super) mapping_version: Version, + pub(super) source: ReadableBoxedVec, + #[allow(clippy::type_complexity)] + pub(super) mapping: Arc Arc<[S2T]> + Send + Sync>, + #[allow(clippy::type_complexity)] + pub(super) _phantom: PhantomData (I, O, Strat)>, +} + +impl LazyAggVec +where + I: VecIndex, + O: VecValue, + S1I: VecIndex, + S2T: VecValue, + S1T: VecValue, +{ + pub fn new( + name: &str, + version: Version, + mapping_version: Version, + source: ReadableBoxedVec, + mapping: impl Fn() -> Arc<[S2T]> + Send + Sync + 'static, + ) -> Self { + Self { + name: Arc::from(name), + version, + mapping_version, + source, + mapping: Arc::new(mapping), + _phantom: PhantomData, + } + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/readable.rs b/crates/vecdb/src/variants/lazy/agg/readable.rs new file mode 100644 index 000000000..24afdb81e --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/readable.rs @@ -0,0 +1,73 @@ +use crate::{ReadableVec, VecIndex, VecValue}; + +use super::{AggFold, LazyAggVec}; + +impl ReadableVec for LazyAggVec +where + I: VecIndex, + O: VecValue, + S1I: VecIndex, + S2T: VecValue, + S1T: VecValue, + Strat: AggFold, +{ + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let mapping = (self.mapping)(); + let to = to.min(mapping.len()); + if from >= to { + return; + } + buf.reserve(to - from); + Strat::fold(&*self.source, &mapping, from, to, (), |(), v| buf.push(v)); + } + + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(O)) { + let mapping = (self.mapping)(); + let to = to.min(mapping.len()); + if from >= to { + return; + } + Strat::fold(&*self.source, &mapping, from, to, (), |(), v| f(v)); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, f: F) -> B + where + Self: Sized, + { + let mapping = (self.mapping)(); + let to = to.min(mapping.len()); + if from >= to { + return init; + } + Strat::fold(&*self.source, &mapping, from, to, init, f) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized, + { + let mapping = (self.mapping)(); + let to = to.min(mapping.len()); + if from >= to { + return Ok(init); + } + Strat::try_fold(&*self.source, &mapping, from, to, init, f) + } + + #[inline] + fn collect_one_at(&self, index: usize) -> Option { + let mapping = (self.mapping)(); + if index >= mapping.len() { + return None; + } + Strat::collect_one(&*self.source, &mapping, index) + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/sparse.rs b/crates/vecdb/src/variants/lazy/agg/sparse.rs new file mode 100644 index 000000000..345679cf7 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/sparse.rs @@ -0,0 +1,65 @@ +use crate::{AggFold, ReadableVec, VecIndex, VecValue}; + +/// Sparse aggregation: emits `Option` per output index. +/// +/// `Some(last_value)` when the range contains source elements, +/// `None` when the range is empty. +pub struct Sparse; + +impl AggFold, SI, SI, T> for Sparse { + #[inline] + fn try_fold + ?Sized, B, E, F: FnMut(B, Option) -> Result>( + source: &S, + mapping: &[SI], + from: usize, + to: usize, + init: B, + mut f: F, + ) -> Result { + let source_len = source.len(); + + let mut indices: Vec = Vec::with_capacity(to - from); + let mut slot_map: Vec> = Vec::with_capacity(to - from); + + (from..to).for_each(|idx| { + let current_first = mapping[idx].to_usize(); + let next_first = mapping + .get(idx + 1) + .map(|h| h.to_usize()) + .unwrap_or(source_len); + + if next_first == 0 || current_first >= next_first { + slot_map.push(None); + } else { + slot_map.push(Some(indices.len() as u32)); + indices.push(next_first - 1); + } + }); + + let values = source.read_sorted_at(&indices); + + slot_map.iter().try_fold(init, |acc, slot| match slot { + None => f(acc, None), + &Some(vi) => f(acc, Some(values[vi as usize].clone())), + }) + } + + #[inline] + fn collect_one + ?Sized>( + source: &S, + mapping: &[SI], + index: usize, + ) -> Option> { + let source_len = source.len(); + let current_first = mapping[index].to_usize(); + let next_first = mapping + .get(index + 1) + .map(|h| h.to_usize()) + .unwrap_or(source_len); + + if next_first == 0 || current_first >= next_first { + return Some(None); + } + Some(source.collect_one_at(next_first - 1)) + } +} diff --git a/crates/vecdb/src/variants/lazy/agg/typed.rs b/crates/vecdb/src/variants/lazy/agg/typed.rs new file mode 100644 index 000000000..1be283167 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/agg/typed.rs @@ -0,0 +1,16 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::LazyAggVec; + +impl TypedVec for LazyAggVec +where + I: VecIndex, + O: VecValue, + S1I: VecIndex, + S2T: VecValue, + S1T: VecValue, + Strat: 'static, +{ + type I = I; + type T = O; +} diff --git a/crates/vecdb/src/variants/lazy/delta/any_vec.rs b/crates/vecdb/src/variants/lazy/delta/any_vec.rs new file mode 100644 index 000000000..d5533f7f6 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/any_vec.rs @@ -0,0 +1,42 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::{DeltaOp, LazyDeltaVec}; + +impl AnyVec for LazyDeltaVec +where + I: VecIndex, + S: VecValue, + T: VecValue, + Op: DeltaOp, +{ + fn version(&self) -> Version { + self.base_version + self.source.version() + self.window_starts_version + } + + fn name(&self) -> &str { + &self.name + } + + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + fn len(&self) -> usize { + self.source.len() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + Vec::new() + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/avg.rs b/crates/vecdb/src/variants/lazy/delta/avg.rs new file mode 100644 index 000000000..b8c1ba1a9 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/avg.rs @@ -0,0 +1,35 @@ +use super::DeltaOp; + +/// Rolling average from cumulative: `(cum[h] - cum[start - 1]) / (h - start + 1)` +#[derive(Clone, Copy)] +pub struct DeltaAvg; + +impl DeltaOp for DeltaAvg +where + S: Into + Default, + T: From, +{ + #[inline] + fn ago_index(start: usize) -> Option { + start.checked_sub(1) + } + + #[inline] + fn ago_default() -> S { + S::default() + } + + #[inline] + fn count(h: usize, start: usize) -> usize { + h - start + 1 + } + + #[inline] + fn combine(current: S, ago: S, count: usize) -> T { + if count == 0 { + T::from(0.0) + } else { + T::from((current.into() - ago.into()) / count as f64) + } + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/change.rs b/crates/vecdb/src/variants/lazy/delta/change.rs new file mode 100644 index 000000000..9b2bf9586 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/change.rs @@ -0,0 +1,16 @@ +use super::DeltaOp; + +/// Delta change: `source[h] - source[start]` via f64, allowing cross-type (unsigned → signed). +#[derive(Clone, Copy)] +pub struct DeltaChange; + +impl DeltaOp for DeltaChange +where + S: Into, + C: From, +{ + #[inline] + fn combine(current: S, ago: S, _count: usize) -> C { + C::from(Into::::into(current) - Into::::into(ago)) + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/clone.rs b/crates/vecdb/src/variants/lazy/delta/clone.rs new file mode 100644 index 000000000..41ce3b6e5 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/clone.rs @@ -0,0 +1,22 @@ +use std::marker::PhantomData; + +use crate::{VecIndex, VecValue}; + +use super::LazyDeltaVec; + +impl Clone for LazyDeltaVec +where + I: VecIndex, + S: VecValue, +{ + fn clone(&self) -> Self { + Self { + name: self.name.clone(), + base_version: self.base_version, + source: self.source.clone(), + window_starts_version: self.window_starts_version, + window_starts: self.window_starts.clone(), + _op: PhantomData, + } + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/mod.rs b/crates/vecdb/src/variants/lazy/delta/mod.rs new file mode 100644 index 000000000..133064759 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/mod.rs @@ -0,0 +1,114 @@ +use std::{marker::PhantomData, sync::Arc}; + +mod any_vec; +mod avg; +mod change; +mod clone; +mod op; +mod rate; +mod readable; +mod sub; +mod typed; + +pub use avg::DeltaAvg; +pub use change::DeltaChange; +pub use op::DeltaOp; +pub use rate::DeltaRate; +pub use sub::DeltaSub; + +use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; + +/// Lazily computed vector that combines a source value with a lookback value. +/// +/// For each index `h` with `start = window_starts[h]`: +/// - `INCLUSIVE` ops (cumulative source): reads `source[h]` and `source[start - 1]`, +/// count = `h - start + 1`. Used for rolling sums/averages from prefix sums. +/// - Non-inclusive ops (raw source): reads `source[h]` and `source[start]`, +/// count = `h - start`. Used for point-to-point deltas (change, rate). +/// +/// Nothing is stored on disk — values are computed on-the-fly during iteration. +pub struct LazyDeltaVec { + pub(super) name: Arc, + pub(super) base_version: Version, + pub(super) source: ReadableBoxedVec, + pub(super) window_starts_version: Version, + #[allow(clippy::type_complexity)] + pub(super) window_starts: Arc Arc<[I]> + Send + Sync>, + pub(super) _op: PhantomData<(Op, T)>, +} + +impl LazyDeltaVec +where + I: VecIndex, + S: VecValue, + T: VecValue, + Op: DeltaOp, +{ + pub fn new( + name: &str, + version: Version, + source: ReadableBoxedVec, + window_starts_version: Version, + window_starts: impl Fn() -> Arc<[I]> + Send + Sync + 'static, + ) -> Self { + Self { + name: Arc::from(name), + base_version: version, + source, + window_starts_version, + window_starts: Arc::new(window_starts), + _op: PhantomData, + } + } + + /// Core bulk iteration with fold + early exit support: collect source range + /// covering both current and ago positions in a single sequential read, + /// then apply Op per element. + #[inline] + pub(super) fn bulk_try_fold( + &self, + from: usize, + to: usize, + starts: &[I], + init: B, + mut f: impl FnMut(B, T) -> std::result::Result, + ) -> std::result::Result { + if from >= to { + return Ok(init); + } + + // Starts are monotonically non-decreasing, so the earliest ago is from starts[from]. + let read_from = Op::ago_index(starts[from].to_usize()) + .unwrap_or(0) + .min(from); + + let source_data = self.source.collect_range_dyn(read_from, to); + + let mut acc = init; + for i in from..to { + let start = starts[i].to_usize(); + let current = source_data[i - read_from].clone(); + let ago = match Op::ago_index(start) { + Some(idx) => source_data[idx - read_from].clone(), + None => Op::ago_default(), + }; + acc = f(acc, Op::combine(current, ago, Op::count(i, start)))?; + } + Ok(acc) + } + + #[inline] + pub(super) fn bulk_for_each( + &self, + from: usize, + to: usize, + starts: &[I], + mut each: impl FnMut(T), + ) { + self.bulk_try_fold(from, to, starts, (), |(), v| { + each(v); + Ok::<_, std::convert::Infallible>(()) + }) + .unwrap_or_else(|e: std::convert::Infallible| match e {}) + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/op.rs b/crates/vecdb/src/variants/lazy/delta/op.rs new file mode 100644 index 000000000..7f8424f58 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/op.rs @@ -0,0 +1,30 @@ +/// Trait defining how to combine a current value with an earlier value. +/// +/// `S` is the source type read from the vec, `T` is the output type produced. +/// When `S = T` (e.g., rolling sums), the operation is same-type. +/// When `S != T` (e.g., delta change/rate), the operation converts between types. +pub trait DeltaOp: Send + Sync + 'static { + /// Source index for the `ago` value given a window start. + /// Returns `None` when there is no preceding element (cumulative ops at start = 0). + #[inline] + fn ago_index(start: usize) -> Option { + Some(start) + } + + /// Fallback `ago` value when `ago_index` returns `None`. + #[inline] + fn ago_default() -> S + where + S: Sized, + { + unreachable!() + } + + /// Window element count from current index `h` and window start. + #[inline] + fn count(h: usize, start: usize) -> usize { + h - start + } + + fn combine(current: S, ago: S, count: usize) -> T; +} diff --git a/crates/vecdb/src/variants/lazy/delta/rate.rs b/crates/vecdb/src/variants/lazy/delta/rate.rs new file mode 100644 index 000000000..865ec069c --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/rate.rs @@ -0,0 +1,22 @@ +use super::DeltaOp; + +/// Delta rate (growth): `(source[h] - source[start]) / source[start]` via f64. +#[derive(Clone, Copy)] +pub struct DeltaRate; + +impl DeltaOp for DeltaRate +where + S: Into, + B: From, +{ + #[inline] + fn combine(current: S, ago: S, _count: usize) -> B { + let current_f: f64 = current.into(); + let ago_f: f64 = ago.into(); + if ago_f == 0.0 { + B::from(0.0) + } else { + B::from((current_f - ago_f) / ago_f) + } + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/readable.rs b/crates/vecdb/src/variants/lazy/delta/readable.rs new file mode 100644 index 000000000..2a3afed09 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/readable.rs @@ -0,0 +1,145 @@ +use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; + +use super::{DeltaOp, LazyDeltaVec}; + +impl ReadableVec for LazyDeltaVec +where + I: VecIndex, + S: VecValue, + T: VecValue, + Op: DeltaOp, +{ + #[inline] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let starts = (self.window_starts)(); + let to = to.min(self.len()).min(starts.len()); + if from >= to { + return; + } + buf.reserve(to - from); + self.bulk_for_each(from, to, &starts, |v| buf.push(v)); + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + let starts = (self.window_starts)(); + let to = to.min(self.len()).min(starts.len()); + if from >= to { + return; + } + self.bulk_for_each(from, to, &starts, f); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + Self: Sized, + { + let starts = (self.window_starts)(); + let to = to.min(self.len()).min(starts.len()); + if from >= to { + return init; + } + self.bulk_try_fold(from, to, &starts, init, |acc, v| { + Ok::<_, std::convert::Infallible>(f(acc, v)) + }) + .unwrap_or_else(|e: std::convert::Infallible| match e {}) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized, + { + let starts = (self.window_starts)(); + let to = to.min(self.len()).min(starts.len()); + if from >= to { + return Ok(init); + } + self.bulk_try_fold(from, to, &starts, init, f) + } + + #[inline] + fn collect_one_at(&self, index: usize) -> Option { + if index >= self.len() { + return None; + } + let starts = (self.window_starts)(); + if index >= starts.len() { + return None; + } + let start = starts[index].to_usize(); + let current = self.source.collect_one_at(index)?; + let ago = match Op::ago_index(start) { + Some(idx) => self.source.collect_one_at(idx)?, + None => Op::ago_default(), + }; + Some(Op::combine(current, ago, Op::count(index, start))) + } + + fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { + if indices.is_empty() { + return; + } + + let starts = (self.window_starts)(); + let len = self.len().min(starts.len()); + let count = indices.len(); + + let mut reads: Vec<(usize, u32, bool)> = Vec::with_capacity(count * 2); + indices.iter().enumerate().for_each(|(slot, &h)| { + if h < len { + reads.push((h, slot as u32, true)); + if let Some(ago_idx) = Op::ago_index(starts[h].to_usize()) { + reads.push((ago_idx, slot as u32, false)); + } + } + }); + reads.sort_unstable_by_key(|r| r.0); + + let mut positions: Vec = Vec::with_capacity(reads.len()); + let mut val_indices: Vec = Vec::with_capacity(reads.len()); + reads.iter().for_each(|&(pos, _, _)| { + if positions.last() != Some(&pos) { + positions.push(pos); + } + val_indices.push((positions.len() - 1) as u32); + }); + + let vals = self.source.read_sorted_at(&positions); + + let mut current_vi = vec![0u32; count]; + let mut ago_vi = vec![0u32; count]; + reads + .iter() + .enumerate() + .for_each(|(i, &(_, slot, is_current))| { + let vi = val_indices[i]; + if is_current { + current_vi[slot as usize] = vi; + } else { + ago_vi[slot as usize] = vi; + } + }); + + out.reserve(count); + indices.iter().enumerate().for_each(|(slot, &h)| { + if h >= len { + return; + } + let start = starts[h].to_usize(); + let current = vals[current_vi[slot] as usize].clone(); + let ago = match Op::ago_index(start) { + Some(_) => vals[ago_vi[slot] as usize].clone(), + None => Op::ago_default(), + }; + out.push(Op::combine(current, ago, Op::count(h, start))); + }); + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/sub.rs b/crates/vecdb/src/variants/lazy/delta/sub.rs new file mode 100644 index 000000000..66f26859b --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/sub.rs @@ -0,0 +1,32 @@ +use crate::CheckedSub; + +use super::DeltaOp; + +/// Rolling sum from cumulative: `cum[h] - cum[start - 1]` +#[derive(Clone, Copy)] +pub struct DeltaSub; + +impl DeltaOp for DeltaSub +where + T: CheckedSub + Default, +{ + #[inline] + fn ago_index(start: usize) -> Option { + start.checked_sub(1) + } + + #[inline] + fn ago_default() -> T { + T::default() + } + + #[inline] + fn count(h: usize, start: usize) -> usize { + h - start + 1 + } + + #[inline] + fn combine(current: T, ago: T, _count: usize) -> T { + current.checked_sub(ago).unwrap_or_default() + } +} diff --git a/crates/vecdb/src/variants/lazy/delta/typed.rs b/crates/vecdb/src/variants/lazy/delta/typed.rs new file mode 100644 index 000000000..6d5d823ab --- /dev/null +++ b/crates/vecdb/src/variants/lazy/delta/typed.rs @@ -0,0 +1,14 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::{DeltaOp, LazyDeltaVec}; + +impl TypedVec for LazyDeltaVec +where + I: VecIndex, + S: VecValue, + T: VecValue, + Op: DeltaOp, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/lazy/from1/any_vec.rs b/crates/vecdb/src/variants/lazy/from1/any_vec.rs new file mode 100644 index 000000000..6faddc593 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/any_vec.rs @@ -0,0 +1,42 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::LazyVecFrom1; + +impl AnyVec for LazyVecFrom1 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, +{ + fn version(&self) -> Version { + self.base_version + self.source.version() + } + + fn name(&self) -> &str { + &self.name + } + + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + fn len(&self) -> usize { + self.source.len() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + Vec::new() + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/mod.rs b/crates/vecdb/src/variants/lazy/from1/mod.rs new file mode 100644 index 000000000..2728f1973 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/mod.rs @@ -0,0 +1,81 @@ +use std::sync::Arc; + +mod any_vec; +mod read_only_clone; +mod readable; +mod transform; +mod typed; + +pub use transform::*; + +use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; + +pub type ComputeFrom1 = fn(I, S1T) -> T; + +/// Lazily computed vector deriving values on-the-fly from one source vector. +/// +/// Unlike `EagerVec`, no data is stored on disk. Values are computed during +/// iteration by applying a function to the source vector's elements. Use when: +/// - Storage space is limited +/// - Computation is cheap relative to disk I/O +/// - Values are only accessed once or infrequently +/// +/// For frequently accessed derived data, prefer `EagerVec` for better performance. +#[derive(Clone)] +pub struct LazyVecFrom1 +where + S1I: VecIndex, + S1T: VecValue, +{ + pub(super) name: Arc, + pub(super) base_version: Version, + pub(super) source: ReadableBoxedVec, + pub(super) compute: ComputeFrom1, +} + +impl LazyVecFrom1 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, +{ + pub fn init( + name: &str, + version: Version, + source: ReadableBoxedVec, + compute: ComputeFrom1, + ) -> Self { + assert_eq!( + I::to_string(), + S1I::to_string(), + "LazyVecFrom1 index type mismatch: expected {}, got {}", + I::to_string(), + S1I::to_string() + ); + + Self { + name: Arc::from(name), + base_version: version, + source, + compute, + } + } +} + +impl LazyVecFrom1 +where + I: VecIndex, + T: VecValue, + S1T: VecValue, +{ + /// Create a lazy vec with a generic transform. + /// Usage: `LazyVecFrom1::transformed::(name, v, source)` + pub fn transformed>( + name: &str, + version: Version, + source: ReadableBoxedVec, + ) -> Self { + Self::init(name, version, source, |_, v| F::apply(v)) + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/read_only_clone.rs b/crates/vecdb/src/variants/lazy/from1/read_only_clone.rs new file mode 100644 index 000000000..96ea7a4a6 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/read_only_clone.rs @@ -0,0 +1,17 @@ +use crate::{ReadOnlyClone, VecIndex, VecValue}; + +use super::LazyVecFrom1; + +impl ReadOnlyClone for LazyVecFrom1 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, +{ + type ReadOnly = Self; + + fn read_only_clone(&self) -> Self { + self.clone() + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/readable.rs b/crates/vecdb/src/variants/lazy/from1/readable.rs new file mode 100644 index 000000000..5979efabe --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/readable.rs @@ -0,0 +1,80 @@ +use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; + +use super::LazyVecFrom1; + +impl ReadableVec for LazyVecFrom1 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, +{ + #[inline] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let to = to.min(self.len()); + buf.reserve(to.saturating_sub(from)); + self.for_each_range_dyn_at(from, to, &mut |v| buf.push(v)); + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + let compute = self.compute; + let to = to.min(self.len()); + let mut pos = from; + self.source.for_each_range_dyn_at(from, to, &mut |v| { + f(compute(I::from(pos), v)); + pos += 1; + }); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + Self: Sized, + { + self.try_fold_range_at(from, to, init, |acc, v| { + Ok::<_, std::convert::Infallible>(f(acc, v)) + }) + .unwrap_or_else(|e: std::convert::Infallible| match e {}) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result + where + Self: Sized, + { + let to = to.min(self.len()); + if from >= to { + return Ok(init); + } + let compute = self.compute; + let buf = self.source.collect_range_dyn(from, to); + buf.into_iter() + .enumerate() + .try_fold(init, |acc, (local, v)| { + f(acc, compute(I::from(from + local), v)) + }) + } + + #[inline] + fn collect_one_at(&self, index: usize) -> Option { + let v = self.source.collect_one_at(index)?; + Some((self.compute)(I::from(index), v)) + } + + fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { + let compute = self.compute; + let source_vals = self.source.read_sorted_at(indices); + out.reserve(source_vals.len()); + indices + .iter() + .zip(source_vals) + .for_each(|(&i, v)| out.push(compute(I::from(i), v))); + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/transform/halve.rs b/crates/vecdb/src/variants/lazy/from1/transform/halve.rs new file mode 100644 index 000000000..760f38619 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/transform/halve.rs @@ -0,0 +1,13 @@ +use std::ops::Div; + +use super::UnaryTransform; + +/// v -> v / 2 +pub struct Halve; + +impl> UnaryTransform for Halve { + #[inline(always)] + fn apply(value: T) -> T { + value / 2 + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/transform/ident.rs b/crates/vecdb/src/variants/lazy/from1/transform/ident.rs new file mode 100644 index 000000000..aa97f7053 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/transform/ident.rs @@ -0,0 +1,11 @@ +use super::UnaryTransform; + +/// v -> v +pub struct Ident; + +impl UnaryTransform for Ident { + #[inline(always)] + fn apply(value: T) -> T { + value + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/transform/mod.rs b/crates/vecdb/src/variants/lazy/from1/transform/mod.rs new file mode 100644 index 000000000..34118286c --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/transform/mod.rs @@ -0,0 +1,13 @@ +mod halve; +mod ident; +mod negate; + +pub use halve::Halve; +pub use ident::Ident; +pub use negate::Negate; + +/// Trait for unary transforms applied lazily during iteration. +/// Zero-sized types implementing this get monomorphized (zero runtime cost). +pub trait UnaryTransform { + fn apply(value: In) -> Out; +} diff --git a/crates/vecdb/src/variants/lazy/from1/transform/negate.rs b/crates/vecdb/src/variants/lazy/from1/transform/negate.rs new file mode 100644 index 000000000..769f2e51d --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/transform/negate.rs @@ -0,0 +1,13 @@ +use std::ops::Neg; + +use super::UnaryTransform; + +/// v -> -v +pub struct Negate; + +impl> UnaryTransform for Negate { + #[inline(always)] + fn apply(value: T) -> T { + -value + } +} diff --git a/crates/vecdb/src/variants/lazy/from1/typed.rs b/crates/vecdb/src/variants/lazy/from1/typed.rs new file mode 100644 index 000000000..b7a090674 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from1/typed.rs @@ -0,0 +1,14 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::LazyVecFrom1; + +impl TypedVec for LazyVecFrom1 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/lazy/from2/any_vec.rs b/crates/vecdb/src/variants/lazy/from2/any_vec.rs new file mode 100644 index 000000000..44eeb4588 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/any_vec.rs @@ -0,0 +1,54 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::LazyVecFrom2; + +impl AnyVec for LazyVecFrom2 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, +{ + fn version(&self) -> Version { + self.base_version + self.source1.version() + self.source2.version() + } + + fn name(&self) -> &str { + &self.name + } + + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + fn len(&self) -> usize { + let len1 = if self.s1_counts { + self.source1.len() + } else { + usize::MAX + }; + let len2 = if self.s2_counts { + self.source2.len() + } else { + usize::MAX + }; + len1.min(len2) + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + Vec::new() + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/mod.rs b/crates/vecdb/src/variants/lazy/from2/mod.rs new file mode 100644 index 000000000..681f1a107 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/mod.rs @@ -0,0 +1,95 @@ +use std::sync::Arc; + +mod any_vec; +mod readable; +mod transform; +mod typed; + +pub use transform::*; + +use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; + +pub type ComputeFrom2 = fn(I, S1T, S2T) -> T; + +/// Lazily computed vector deriving values from two source vectors. +/// +/// Values are computed on-the-fly during iteration using a provided function. +/// Nothing is stored on disk - all values are recomputed each time they're accessed. +#[derive(Clone)] +pub struct LazyVecFrom2 +where + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, +{ + pub(super) name: Arc, + pub(super) base_version: Version, + pub(super) source1: ReadableBoxedVec, + pub(super) source2: ReadableBoxedVec, + pub(super) compute: ComputeFrom2, + pub(super) s1_counts: bool, + pub(super) s2_counts: bool, +} + +impl LazyVecFrom2 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, +{ + pub fn init( + name: &str, + version: Version, + source1: ReadableBoxedVec, + source2: ReadableBoxedVec, + compute: ComputeFrom2, + ) -> Self { + let target = I::to_string(); + let s1 = source1.index_type_to_string(); + let s2 = source2.index_type_to_string(); + + assert!( + s1 == target || s2 == target, + "LazyVecFrom2: at least one source must have index type {}, got {} and {}", + target, + s1, + s2 + ); + + let s1_counts = s1 == target; + let s2_counts = s2 == target; + + Self { + name: Arc::from(name), + base_version: version, + source1, + source2, + compute, + s1_counts, + s2_counts, + } + } +} + +impl LazyVecFrom2 +where + I: VecIndex, + T: VecValue, + S1T: VecValue, + S2T: VecValue, +{ + /// Create a lazy vec with a generic binary transform. + /// Usage: `LazyVecFrom2::transformed::(name, v, source1, source2)` + pub fn transformed>( + name: &str, + version: Version, + source1: ReadableBoxedVec, + source2: ReadableBoxedVec, + ) -> Self { + Self::init(name, version, source1, source2, |_, a, b| F::apply(a, b)) + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/readable.rs b/crates/vecdb/src/variants/lazy/from2/readable.rs new file mode 100644 index 000000000..5b3adec78 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/readable.rs @@ -0,0 +1,92 @@ +use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; + +use super::LazyVecFrom2; + +impl ReadableVec for LazyVecFrom2 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, +{ + #[inline] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let to = to.min(self.len()); + buf.reserve(to.saturating_sub(from)); + self.for_each_range_dyn_at(from, to, &mut |v| buf.push(v)); + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + let compute = self.compute; + let to = to.min(self.len()); + let buf1 = self.source1.collect_range_dyn(from, to); + let buf2 = self.source2.collect_range_dyn(from, to); + buf1.into_iter() + .zip(buf2) + .enumerate() + .for_each(|(local, (v1, v2))| { + f(compute(I::from(from + local), v1, v2)); + }); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + Self: Sized, + { + self.try_fold_range_at(from, to, init, |acc, v| { + Ok::<_, std::convert::Infallible>(f(acc, v)) + }) + .unwrap_or_else(|e: std::convert::Infallible| match e {}) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result + where + Self: Sized, + { + let to = to.min(self.len()); + if from >= to { + return Ok(init); + } + let compute = self.compute; + let buf1 = self.source1.collect_range_dyn(from, to); + let buf2 = self.source2.collect_range_dyn(from, to); + buf1.into_iter() + .zip(buf2) + .enumerate() + .try_fold(init, |acc, (local, (v1, v2))| { + f(acc, compute(I::from(from + local), v1, v2)) + }) + } + + #[inline] + fn collect_one_at(&self, index: usize) -> Option { + if index >= self.len() { + return None; + } + let v1 = self.source1.collect_one_at(index)?; + let v2 = self.source2.collect_one_at(index)?; + Some((self.compute)(I::from(index), v1, v2)) + } + + fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { + let compute = self.compute; + let vals1 = self.source1.read_sorted_at(indices); + let vals2 = self.source2.read_sorted_at(indices); + out.reserve(vals1.len().min(vals2.len())); + indices + .iter() + .zip(vals1.into_iter().zip(vals2)) + .for_each(|(&i, (v1, v2))| out.push(compute(I::from(i), v1, v2))); + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/transform/divide.rs b/crates/vecdb/src/variants/lazy/from2/transform/divide.rs new file mode 100644 index 000000000..5bd3719f8 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/transform/divide.rs @@ -0,0 +1,13 @@ +use std::ops::Div; + +use super::BinaryTransform; + +/// (a, b) -> a / b +pub struct Divide; + +impl, U, O> BinaryTransform for Divide { + #[inline(always)] + fn apply(lhs: T, rhs: U) -> O { + lhs / rhs + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/transform/minus.rs b/crates/vecdb/src/variants/lazy/from2/transform/minus.rs new file mode 100644 index 000000000..487da7e3d --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/transform/minus.rs @@ -0,0 +1,13 @@ +use std::ops::Sub; + +use super::BinaryTransform; + +/// (a, b) -> a - b +pub struct Minus; + +impl, U, O> BinaryTransform for Minus { + #[inline(always)] + fn apply(lhs: T, rhs: U) -> O { + lhs - rhs + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/transform/mod.rs b/crates/vecdb/src/variants/lazy/from2/transform/mod.rs new file mode 100644 index 000000000..8019fe52f --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/transform/mod.rs @@ -0,0 +1,15 @@ +mod divide; +mod minus; +mod plus; +mod times; + +pub use divide::Divide; +pub use minus::Minus; +pub use plus::Plus; +pub use times::Times; + +/// Trait for binary transforms applied lazily during iteration. +/// Zero-sized types implementing this get monomorphized (zero runtime cost). +pub trait BinaryTransform { + fn apply(lhs: In1, rhs: In2) -> Out; +} diff --git a/crates/vecdb/src/variants/lazy/from2/transform/plus.rs b/crates/vecdb/src/variants/lazy/from2/transform/plus.rs new file mode 100644 index 000000000..5db4bb9a9 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/transform/plus.rs @@ -0,0 +1,13 @@ +use std::ops::Add; + +use super::BinaryTransform; + +/// (a, b) -> a + b +pub struct Plus; + +impl, U, O> BinaryTransform for Plus { + #[inline(always)] + fn apply(lhs: T, rhs: U) -> O { + lhs + rhs + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/transform/times.rs b/crates/vecdb/src/variants/lazy/from2/transform/times.rs new file mode 100644 index 000000000..7876c9f71 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/transform/times.rs @@ -0,0 +1,13 @@ +use std::ops::Mul; + +use super::BinaryTransform; + +/// (a, b) -> a * b +pub struct Times; + +impl, U, O> BinaryTransform for Times { + #[inline(always)] + fn apply(lhs: T, rhs: U) -> O { + lhs * rhs + } +} diff --git a/crates/vecdb/src/variants/lazy/from2/typed.rs b/crates/vecdb/src/variants/lazy/from2/typed.rs new file mode 100644 index 000000000..27c8e05a9 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from2/typed.rs @@ -0,0 +1,16 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::LazyVecFrom2; + +impl TypedVec for LazyVecFrom2 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/lazy/from3/any_vec.rs b/crates/vecdb/src/variants/lazy/from3/any_vec.rs new file mode 100644 index 000000000..5b5ab6dfb --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from3/any_vec.rs @@ -0,0 +1,61 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::LazyVecFrom3; + +impl AnyVec for LazyVecFrom3 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, + S3I: VecIndex, + S3T: VecValue, +{ + fn version(&self) -> Version { + self.base_version + self.source1.version() + self.source2.version() + self.source3.version() + } + + fn name(&self) -> &str { + &self.name + } + + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + fn len(&self) -> usize { + let len1 = if self.s1_counts { + self.source1.len() + } else { + usize::MAX + }; + let len2 = if self.s2_counts { + self.source2.len() + } else { + usize::MAX + }; + let len3 = if self.s3_counts { + self.source3.len() + } else { + usize::MAX + }; + len1.min(len2).min(len3) + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + Vec::new() + } +} diff --git a/crates/vecdb/src/variants/lazy/from3/mod.rs b/crates/vecdb/src/variants/lazy/from3/mod.rs new file mode 100644 index 000000000..aa31645f6 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from3/mod.rs @@ -0,0 +1,85 @@ +use std::sync::Arc; + +mod any_vec; +mod readable; +mod typed; + +use crate::{ReadableBoxedVec, VecIndex, VecValue, Version}; + +pub type ComputeFrom3 = fn(I, S1T, S2T, S3T) -> T; + +/// Lazily computed vector deriving values from three source vectors. +/// +/// Values are computed on-the-fly during iteration using a provided function. +/// Nothing is stored on disk - all values are recomputed each time they're accessed. +#[derive(Clone)] +pub struct LazyVecFrom3 +where + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, + S3I: VecIndex, + S3T: VecValue, +{ + pub(super) name: Arc, + pub(super) base_version: Version, + pub(super) source1: ReadableBoxedVec, + pub(super) source2: ReadableBoxedVec, + pub(super) source3: ReadableBoxedVec, + pub(super) compute: ComputeFrom3, + pub(super) s1_counts: bool, + pub(super) s2_counts: bool, + pub(super) s3_counts: bool, +} + +impl LazyVecFrom3 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, + S3I: VecIndex, + S3T: VecValue, +{ + pub fn init( + name: &str, + version: Version, + source1: ReadableBoxedVec, + source2: ReadableBoxedVec, + source3: ReadableBoxedVec, + compute: ComputeFrom3, + ) -> Self { + let target = I::to_string(); + let s1 = source1.index_type_to_string(); + let s2 = source2.index_type_to_string(); + let s3 = source3.index_type_to_string(); + + assert!( + s1 == target || s2 == target || s3 == target, + "LazyVecFrom3: at least one source must have index type {}, got {}, {}, and {}", + target, + s1, + s2, + s3 + ); + + let s1_counts = s1 == target; + let s2_counts = s2 == target; + let s3_counts = s3 == target; + + Self { + name: Arc::from(name), + base_version: version, + source1, + source2, + source3, + compute, + s1_counts, + s2_counts, + s3_counts, + } + } +} diff --git a/crates/vecdb/src/variants/lazy/from3/readable.rs b/crates/vecdb/src/variants/lazy/from3/readable.rs new file mode 100644 index 000000000..c2f607d53 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from3/readable.rs @@ -0,0 +1,101 @@ +use crate::{AnyVec, ReadableVec, VecIndex, VecValue}; + +use super::LazyVecFrom3; + +impl ReadableVec + for LazyVecFrom3 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, + S3I: VecIndex, + S3T: VecValue, +{ + #[inline] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let to = to.min(self.len()); + buf.reserve(to.saturating_sub(from)); + self.for_each_range_dyn_at(from, to, &mut |v| buf.push(v)); + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + let compute = self.compute; + let to = to.min(self.len()); + let buf1 = self.source1.collect_range_dyn(from, to); + let buf2 = self.source2.collect_range_dyn(from, to); + let buf3 = self.source3.collect_range_dyn(from, to); + buf1.into_iter() + .zip(buf2) + .zip(buf3) + .enumerate() + .for_each(|(local, ((v1, v2), v3))| { + f(compute(I::from(from + local), v1, v2, v3)); + }); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + Self: Sized, + { + self.try_fold_range_at(from, to, init, |acc, v| { + Ok::<_, std::convert::Infallible>(f(acc, v)) + }) + .unwrap_or_else(|e: std::convert::Infallible| match e {}) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result + where + Self: Sized, + { + let to = to.min(self.len()); + if from >= to { + return Ok(init); + } + let compute = self.compute; + let buf1 = self.source1.collect_range_dyn(from, to); + let buf2 = self.source2.collect_range_dyn(from, to); + let buf3 = self.source3.collect_range_dyn(from, to); + buf1.into_iter() + .zip(buf2) + .zip(buf3) + .enumerate() + .try_fold(init, |acc, (local, ((v1, v2), v3))| { + f(acc, compute(I::from(from + local), v1, v2, v3)) + }) + } + + #[inline] + fn collect_one_at(&self, index: usize) -> Option { + if index >= self.len() { + return None; + } + let v1 = self.source1.collect_one_at(index)?; + let v2 = self.source2.collect_one_at(index)?; + let v3 = self.source3.collect_one_at(index)?; + Some((self.compute)(I::from(index), v1, v2, v3)) + } + + fn read_sorted_into_at(&self, indices: &[usize], out: &mut Vec) { + let compute = self.compute; + let vals1 = self.source1.read_sorted_at(indices); + let vals2 = self.source2.read_sorted_at(indices); + let vals3 = self.source3.read_sorted_at(indices); + out.reserve(vals1.len().min(vals2.len()).min(vals3.len())); + indices + .iter() + .zip(vals1.into_iter().zip(vals2).zip(vals3)) + .for_each(|(&i, ((v1, v2), v3))| out.push(compute(I::from(i), v1, v2, v3))); + } +} diff --git a/crates/vecdb/src/variants/lazy/from3/typed.rs b/crates/vecdb/src/variants/lazy/from3/typed.rs new file mode 100644 index 000000000..84fc50603 --- /dev/null +++ b/crates/vecdb/src/variants/lazy/from3/typed.rs @@ -0,0 +1,19 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::LazyVecFrom3; + +impl TypedVec + for LazyVecFrom3 +where + I: VecIndex, + T: VecValue, + S1I: VecIndex, + S1T: VecValue, + S2I: VecIndex, + S2T: VecValue, + S3I: VecIndex, + S3T: VecValue, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/lazy/mod.rs b/crates/vecdb/src/variants/lazy/mod.rs new file mode 100644 index 000000000..d017f44ef --- /dev/null +++ b/crates/vecdb/src/variants/lazy/mod.rs @@ -0,0 +1,11 @@ +mod agg; +mod delta; +mod from1; +mod from2; +mod from3; + +pub use agg::*; +pub use delta::*; +pub use from1::*; +pub use from2::*; +pub use from3::*; diff --git a/crates/vecdb/src/variants/macros.rs b/crates/vecdb/src/variants/macros.rs new file mode 100644 index 000000000..97a395401 --- /dev/null +++ b/crates/vecdb/src/variants/macros.rs @@ -0,0 +1,332 @@ +/// Generates trait implementations for vec wrappers (LZ4Vec, PcoVec, ZstdVec, BytesVec, ZeroCopyVec). +/// +/// # Usage +/// ```ignore +/// impl_vec_wrapper!( +/// LZ4Vec, +/// ReadWriteCompressedVec>, +/// LZ4VecValue, +/// Format::LZ4, +/// ); +/// ``` +/// +/// This generates implementations for: +/// - `Deref` / `DerefMut` +/// - `ImportableVec` +/// - `AnyVec` +/// - `TypedVec` +/// - `AnyStoredVec` +/// - `WritableVec` +/// - `ReadableVec` (delegates `for_each_range_dyn` / `fold_range` to inner) +macro_rules! impl_vec_wrapper { + ($wrapper:ident, $inner:ty, $value_trait:ident, $format:expr, $read_only:ty) => { + impl ::std::ops::Deref for $wrapper { + type Target = $inner; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl ::std::ops::DerefMut for $wrapper { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } + } + + impl $crate::ImportableVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + fn import( + db: &::rawdb::Database, + name: &str, + version: $crate::Version, + ) -> $crate::Result { + Self::import_with((db, name, version).into()) + } + + fn import_with(options: $crate::ImportOptions) -> $crate::Result { + Ok(Self(<$inner>::import_with(options, $format)?)) + } + + fn forced_import( + db: &::rawdb::Database, + name: &str, + version: $crate::Version, + ) -> $crate::Result { + Self::forced_import_with((db, name, version).into()) + } + + fn forced_import_with(options: $crate::ImportOptions) -> $crate::Result { + Ok(Self(<$inner>::forced_import_with(options, $format)?)) + } + } + + impl $crate::AnyVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + #[inline] + fn version(&self) -> $crate::Version { + self.0.version() + } + + #[inline] + fn name(&self) -> &str { + self.0.name() + } + + #[inline] + fn len(&self) -> usize { + self.0.len() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + self.0.index_type_to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + self.0.value_type_to_size_of() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + self.0.value_type_to_string() + } + + #[inline] + fn region_names(&self) -> Vec { + self.0.region_names() + } + } + + impl $crate::TypedVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + type I = I; + type T = T; + } + + impl $crate::AnyStoredVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + #[inline] + fn db_path(&self) -> ::std::path::PathBuf { + self.0.db_path() + } + + #[inline] + fn region(&self) -> &::rawdb::Region { + self.0.region() + } + + #[inline] + fn header(&self) -> &$crate::Header { + self.0.header() + } + + #[inline] + fn mut_header(&mut self) -> &mut $crate::Header { + self.0.mut_header() + } + + #[inline] + fn saved_stamped_changes(&self) -> u16 { + self.0.saved_stamped_changes() + } + + #[inline] + fn db(&self) -> ::rawdb::Database { + self.0.db() + } + + #[inline] + fn real_stored_len(&self) -> usize { + self.0.real_stored_len() + } + + #[inline] + fn stored_len(&self) -> usize { + self.0.stored_len() + } + + #[inline] + fn write(&mut self) -> $crate::Result { + self.0.write() + } + + #[inline] + fn serialize_changes(&self) -> $crate::Result> { + self.0.serialize_changes() + } + + #[inline] + fn any_stamped_write_with_changes( + &mut self, + stamp: $crate::Stamp, + ) -> $crate::Result<()> { + $crate::WritableVec::stamped_write_with_changes(&mut self.0, stamp) + } + + fn remove(self) -> $crate::Result<()> { + self.0.remove() + } + + fn any_truncate_if_needed_at(&mut self, index: usize) -> $crate::Result<()> { + $crate::WritableVec::truncate_if_needed_at(&mut self.0, index) + } + + fn any_reset(&mut self) -> $crate::Result<()> { + $crate::WritableVec::reset(self) + } + } + + impl $crate::WritableVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + #[inline] + fn push(&mut self, value: T) { + $crate::WritableVec::push(&mut self.0, value) + } + + #[inline] + fn pushed(&self) -> &[T] { + $crate::WritableVec::pushed(&self.0) + } + + #[inline] + fn truncate_if_needed_at(&mut self, index: usize) -> $crate::Result<()> { + self.0.truncate_if_needed_at(index) + } + + #[inline] + fn reset(&mut self) -> $crate::Result<()> { + self.0.reset() + } + + #[inline] + fn reset_unsaved(&mut self) { + self.0.reset_unsaved() + } + + #[inline] + fn is_dirty(&self) -> bool { + self.0.is_dirty() + } + + #[inline] + fn stamped_write_with_changes(&mut self, stamp: $crate::Stamp) -> $crate::Result<()> { + self.0.stamped_write_with_changes(stamp) + } + + #[inline] + fn rollback(&mut self) -> $crate::Result<()> { + self.0.rollback() + } + + fn find_rollback_files( + &self, + ) -> $crate::Result<::std::collections::BTreeMap<$crate::Stamp, ::std::path::PathBuf>> + { + self.0.find_rollback_files() + } + + fn save_rollback_state(&mut self) { + self.0.save_rollback_state() + } + } + + impl $crate::ReadableCloneableVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + #[inline] + fn read_only_boxed_clone(&self) -> $crate::ReadableBoxedVec { + Box::new(self.0.read_only_clone()) + } + } + + impl $crate::StoredVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + type ReadOnly = $read_only; + + #[inline] + fn read_only_clone(&self) -> Self::ReadOnly { + self.0.read_only_clone() + } + } + + impl $crate::ReadableVec for $wrapper + where + I: $crate::VecIndex, + T: $value_trait, + { + #[inline(always)] + fn cursor_chunk_size(&self) -> usize { + $crate::ReadableVec::::cursor_chunk_size(&self.0) + } + + #[inline(always)] + fn collect_one_at(&self, index: usize) -> Option { + $crate::ReadableVec::::collect_one_at(&self.0, index) + } + + #[inline(always)] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + $crate::ReadableVec::::read_into_at(&self.0, from, to, buf) + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + $crate::ReadableVec::::for_each_range_dyn_at(&self.0, from, to, f) + } + + #[inline] + fn fold_range_at B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B + where + Self: Sized, + { + $crate::ReadableVec::::fold_range_at(&self.0, from, to, init, f) + } + + #[inline] + fn try_fold_range_at ::std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> ::std::result::Result + where + Self: Sized, + { + $crate::ReadableVec::::try_fold_range_at(&self.0, from, to, init, f) + } + } + }; +} + +pub(crate) use impl_vec_wrapper; diff --git a/crates/vecdb/src/variants/mod.rs b/crates/vecdb/src/variants/mod.rs new file mode 100644 index 000000000..fd5a8ac14 --- /dev/null +++ b/crates/vecdb/src/variants/mod.rs @@ -0,0 +1,14 @@ +mod cached; +mod compressed; +mod eager; +mod lazy; +mod macros; +mod raw; + +pub use cached::*; +pub use compressed::*; +pub use eager::*; +pub use lazy::*; +#[allow(unused_imports)] +pub use macros::*; +pub use raw::*; diff --git a/crates/vecdb/src/variants/raw/bytes/mod.rs b/crates/vecdb/src/variants/raw/bytes/mod.rs new file mode 100644 index 000000000..1f801ce6d --- /dev/null +++ b/crates/vecdb/src/variants/raw/bytes/mod.rs @@ -0,0 +1,45 @@ +use crate::{Format, ReadOnlyRawVec, impl_vec_wrapper}; + +use super::ReadWriteRawVec; + +mod strategy; +mod value; + +pub use strategy::*; +pub use value::*; + +/// Raw storage vector using explicit byte serialization in little-endian format. +/// +/// Uses the `Bytes` trait to serialize values with `to_bytes()/from_bytes()` in +/// **LITTLE-ENDIAN** format, ensuring **portability across different endianness systems**. +/// +/// Like `ZeroCopyVec`, this wraps `ReadWriteRawVec` and supports: +/// - Holes (deleted indices) +/// - Updated values (modifications to stored data) +/// - Push/rollback operations +/// +/// The only difference from `ZeroCopyVec` is the serialization strategy: +/// - `BytesVec`: Explicit little-endian, portable across architectures +/// - `ZeroCopyVec`: Native byte order, faster but not portable +/// +/// Use `BytesVec` when: +/// - Sharing data between systems with different endianness +/// - Cross-platform compatibility is required +/// - Custom serialization logic is needed +/// +/// Use `ZeroCopyVec` when: +/// - Maximum performance is critical +/// - Data stays on the same architecture +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct BytesVec(pub(crate) ReadWriteRawVec>); + +pub type BytesVecReader = super::VecReader>; + +impl_vec_wrapper!( + BytesVec, + ReadWriteRawVec>, + BytesVecValue, + Format::Bytes, + ReadOnlyRawVec> +); diff --git a/crates/vecdb/src/variants/raw/bytes/strategy/mod.rs b/crates/vecdb/src/variants/raw/bytes/strategy/mod.rs new file mode 100644 index 000000000..d02c9fa84 --- /dev/null +++ b/crates/vecdb/src/variants/raw/bytes/strategy/mod.rs @@ -0,0 +1,10 @@ +use std::marker::PhantomData; + +mod raw; +mod value; + +/// Serialization strategy using the Bytes trait with portable byte order. +/// +/// Implements little-endian serialization for cross-platform compatibility. +#[derive(Debug, Clone, Copy)] +pub struct BytesStrategy(PhantomData); diff --git a/crates/vecdb/src/variants/raw/bytes/strategy/raw.rs b/crates/vecdb/src/variants/raw/bytes/strategy/raw.rs new file mode 100644 index 000000000..17c65c8b1 --- /dev/null +++ b/crates/vecdb/src/variants/raw/bytes/strategy/raw.rs @@ -0,0 +1,17 @@ +use crate::{BytesVecValue, ValueStrategy, variants::raw::RawStrategy}; + +use super::BytesStrategy; + +impl RawStrategy for BytesStrategy { + #[inline(always)] + unsafe fn read_from_ptr(ptr: *const u8, byte_offset: usize) -> T { + unsafe { + if T::IS_NATIVE_LAYOUT { + (ptr.add(byte_offset) as *const T).read_unaligned() + } else { + let slice = std::slice::from_raw_parts(ptr.add(byte_offset), size_of::()); + as ValueStrategy>::read(slice).unwrap_unchecked() + } + } + } +} diff --git a/crates/vecdb/src/variants/raw/bytes/strategy/value.rs b/crates/vecdb/src/variants/raw/bytes/strategy/value.rs new file mode 100644 index 000000000..96d44549b --- /dev/null +++ b/crates/vecdb/src/variants/raw/bytes/strategy/value.rs @@ -0,0 +1,22 @@ +use crate::{BytesVecValue, Result, ValueStrategy}; + +use super::BytesStrategy; + +impl ValueStrategy for BytesStrategy { + const IS_NATIVE_LAYOUT: bool = T::IS_NATIVE_LAYOUT; + + #[inline(always)] + fn read(bytes: &[u8]) -> Result { + T::from_bytes(bytes) + } + + #[inline(always)] + fn write_to_vec(value: &T, buf: &mut Vec) { + buf.extend_from_slice(value.to_bytes().as_ref()); + } + + #[inline(always)] + fn write_to_slice(value: &T, dst: &mut [u8]) { + dst.copy_from_slice(value.to_bytes().as_ref()); + } +} diff --git a/crates/vecdb/src/variants/raw/bytes/value.rs b/crates/vecdb/src/variants/raw/bytes/value.rs new file mode 100644 index 000000000..05c786840 --- /dev/null +++ b/crates/vecdb/src/variants/raw/bytes/value.rs @@ -0,0 +1,11 @@ +use crate::{Bytes, VecValue}; + +/// Value trait for BytesVec. +/// Extends RawVecValue with Bytes trait for custom serialization. +pub trait BytesVecValue +where + Self: VecValue + Bytes, +{ +} + +impl BytesVecValue for T where T: VecValue + Bytes {} diff --git a/crates/vecdb/src/variants/raw/inner/mod.rs b/crates/vecdb/src/variants/raw/inner/mod.rs new file mode 100644 index 000000000..e2ac0346e --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/mod.rs @@ -0,0 +1,7 @@ +mod read_only; +mod read_write; +mod strategy; + +pub use read_only::*; +pub use read_write::*; +pub use strategy::*; diff --git a/crates/vecdb/src/variants/raw/inner/read_only/any_vec.rs b/crates/vecdb/src/variants/raw/inner/read_only/any_vec.rs new file mode 100644 index 000000000..948cbe793 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_only/any_vec.rs @@ -0,0 +1,45 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name, vec_region_name}; + +use super::{super::RawStrategy, ReadOnlyRawVec}; + +impl AnyVec for ReadOnlyRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + #[inline] + fn version(&self) -> Version { + self.base.version() + } + + #[inline] + fn name(&self) -> &str { + self.base.name() + } + + #[inline] + fn len(&self) -> usize { + self.base.len() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + vec![vec_region_name(self.base.name(), I::to_string())] + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_only/mod.rs b/crates/vecdb/src/variants/raw/inner/read_only/mod.rs new file mode 100644 index 000000000..b28cafb3b --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_only/mod.rs @@ -0,0 +1,116 @@ +use std::marker::PhantomData; + +mod any_vec; +mod readable; +mod typed; + +use crate::{ + Error, HEADER_OFFSET, MMAP_CROSSOVER_BYTES, RawIoSource, RawMmapSource, ReadOnlyBaseVec, + Result, Stamp, VecIndex, VecReader, VecValue, +}; + +use super::RawStrategy; + +/// Lean read-only view of a raw vector (~40 bytes). +/// +/// Carries only the fields needed for disk reads: region, shared length, +/// name/header metadata. No holes, no updated map, no pushed buffer, +/// no rollback state. +/// +/// Created via `ReadWriteRawVec::read_only_clone`. +#[derive(Debug, Clone)] +pub struct ReadOnlyRawVec { + pub(super) base: ReadOnlyBaseVec, + _strategy: PhantomData, +} + +impl ReadOnlyRawVec { + pub(crate) fn new(base: ReadOnlyBaseVec) -> Self { + Self { + base, + _strategy: PhantomData, + } + } +} + +impl ReadOnlyRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + pub(crate) fn region(&self) -> &rawdb::Region { + self.base.region() + } + + pub(crate) fn stored_len(&self) -> usize { + self.base.stored_len() + } + + #[inline] + pub fn stamp(&self) -> Stamp { + self.base.header().stamp() + } + + pub fn reader(&self) -> VecReader { + VecReader::from_read_only(self) + } + + #[inline] + pub fn read_at_once(&self, index: usize) -> Result { + let len = self.base.len(); + if index >= len { + return Err(Error::IndexTooHigh { + index, + len, + name: self.base.name().to_string(), + }); + } + + Ok(self.base.region().with_read_bytes(|bytes| unsafe { + S::read_from_ptr(bytes.as_ptr().add(HEADER_OFFSET), index * size_of::()) + })) + } + + #[inline] + pub fn read_once(&self, index: I) -> Result { + self.read_at_once(index.to_usize()) + } + + #[inline(always)] + pub(super) fn fold_source B>( + &self, + from: usize, + to: usize, + len: usize, + init: B, + f: F, + ) -> B { + let range_bytes = (to - from) * size_of::(); + if range_bytes > MMAP_CROSSOVER_BYTES { + RawIoSource::::new_from_parts(self.base.region(), len, from, to).fold(init, f) + } else { + RawMmapSource::::new_from_parts(self.base.region(), len, from, to) + .fold(init, f) + } + } + + #[inline(always)] + pub(super) fn try_fold_source std::result::Result>( + &self, + from: usize, + to: usize, + len: usize, + init: B, + f: F, + ) -> std::result::Result { + let range_bytes = (to - from) * size_of::(); + if range_bytes > MMAP_CROSSOVER_BYTES { + RawIoSource::::new_from_parts(self.base.region(), len, from, to) + .try_fold(init, f) + } else { + RawMmapSource::::new_from_parts(self.base.region(), len, from, to) + .try_fold(init, f) + } + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_only/readable.rs b/crates/vecdb/src/variants/raw/inner/read_only/readable.rs new file mode 100644 index 000000000..bcba3fa58 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_only/readable.rs @@ -0,0 +1,86 @@ +use crate::{HEADER_OFFSET, ReadableVec, VecIndex, VecValue}; + +use super::{super::RawStrategy, ReadOnlyRawVec}; + +impl ReadableVec for ReadOnlyRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + #[inline(always)] + fn collect_one_at(&self, index: usize) -> Option { + let len = self.base.len(); + if index >= len { + return None; + } + Some(self.base.region().with_read_bytes(|bytes| unsafe { + S::read_from_ptr(bytes.as_ptr().add(HEADER_OFFSET), index * size_of::()) + })) + } + + #[inline(always)] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return; + } + buf.reserve(to - from); + if S::IS_NATIVE_LAYOUT { + let reader = self.base.region().create_reader(); + let src = unsafe { + std::slice::from_raw_parts( + reader + .prefixed(HEADER_OFFSET) + .as_ptr() + .add(from * size_of::()) as *const T, + to - from, + ) + }; + buf.extend_from_slice(src); + } else { + self.fold_source(from, to, len, (), |(), v| buf.push(v)); + } + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + self.fold_range_at(from, to, (), |(), v| f(v)); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, f: F) -> B + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return init; + } + self.fold_source(from, to, len, init, f) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return Ok(init); + } + self.try_fold_source(from, to, len, init, f) + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_only/typed.rs b/crates/vecdb/src/variants/raw/inner/read_only/typed.rs new file mode 100644 index 000000000..487dff366 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_only/typed.rs @@ -0,0 +1,13 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::{super::RawStrategy, ReadOnlyRawVec}; + +impl TypedVec for ReadOnlyRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/any_stored_vec.rs b/crates/vecdb/src/variants/raw/inner/read_write/any_stored_vec.rs new file mode 100644 index 000000000..f0b8da643 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/any_stored_vec.rs @@ -0,0 +1,168 @@ +use std::{mem, path::PathBuf}; + +use log::debug; +use rawdb::{Database, Region, unlikely}; + +use crate::{AnyStoredVec, Bytes, HEADER_OFFSET, Header, Result, Stamp, WritableVec}; + +use super::{super::RawStrategy, ReadWriteRawVec}; + +impl AnyStoredVec for ReadWriteRawVec +where + I: crate::VecIndex, + T: crate::VecValue, + S: RawStrategy, +{ + #[inline] + fn db_path(&self) -> PathBuf { + self.base.db_path() + } + + #[inline] + fn header(&self) -> &Header { + self.base.header() + } + + #[inline] + fn mut_header(&mut self) -> &mut Header { + self.base.mut_header() + } + + #[inline] + fn saved_stamped_changes(&self) -> u16 { + self.base.saved_stamped_changes() + } + + fn db(&self) -> Database { + self.region().db() + } + + #[inline] + fn real_stored_len(&self) -> usize { + (self.region().meta().len() - HEADER_OFFSET) / Self::SIZE_OF_T + } + + #[inline] + fn stored_len(&self) -> usize { + self.base.stored_len() + } + + fn write(&mut self) -> Result { + self.base.write_header_if_needed()?; + + let stored_len = self.stored_len(); + let pushed_len = self.base.pushed().len(); + let real_stored_len = self.real_stored_len(); + // After rollback, stored_len can be > real_stored_len (missing items are in updated map) + let truncated = stored_len < real_stored_len; + let expanded = stored_len > real_stored_len; + let has_new_data = pushed_len != 0; + let has_updated_data = !self.updated().is_empty(); + let has_holes = !self.holes().is_empty(); + let had_holes = self.has_stored_holes; + + if !truncated && !expanded && !has_new_data && !has_updated_data && !has_holes && !had_holes + { + return Ok(false); + } + + let from = stored_len * Self::SIZE_OF_T + HEADER_OFFSET; + + if has_new_data { + // Take the pushed buffer to free its heap allocation after writing. + let taken = mem::take(self.base.mut_pushed()); + if S::IS_NATIVE_LAYOUT { + // Bulk write: memory layout matches serialized format, skip per-value + // serialization entirely. Single memcpy from pushed buffer to mmap. + let bytes = unsafe { + std::slice::from_raw_parts( + taken.as_ptr() as *const u8, + taken.len() * Self::SIZE_OF_T, + ) + }; + self.region().truncate_write(from, bytes)?; + } else { + let mut bytes = Vec::with_capacity(pushed_len * Self::SIZE_OF_T); + for v in &taken { + S::write_to_vec(v, &mut bytes); + } + self.region().truncate_write(from, &bytes)?; + } + self.base.update_stored_len(stored_len + pushed_len); + } else if truncated { + self.region().truncate(from)?; + } + + if has_updated_data { + let updated = self.updated.take_current(); + let region = self.region(); + + if unlikely(expanded) { + // After rollback, updates may extend beyond current disk length. + // Use write_at which handles extension (slower but necessary). + let mut bytes = Vec::with_capacity(Self::SIZE_OF_T); + for (index, value) in updated { + let offset = index * Self::SIZE_OF_T + HEADER_OFFSET; + bytes.clear(); + S::write_to_vec(&value, &mut bytes); + region.write_at(&bytes, offset)?; + } + } else { + // Normal case: write directly to mmap, no intermediate allocations + region.batch_write_each( + updated + .into_iter() + .map(|(index, value)| (index * Self::SIZE_OF_T + HEADER_OFFSET, value)), + Self::SIZE_OF_T, + S::write_to_slice, + ); + } + } + + if has_holes { + self.has_stored_holes = true; + let holes_region = self + .region() + .db() + .create_region_if_needed(&self.holes_region_name())?; + let holes = self.holes(); + let mut bytes = Vec::with_capacity(holes.len() * size_of::()); + for i in holes { + bytes.extend(i.to_bytes()); + } + holes_region.truncate_write(0, &bytes)?; + } else if had_holes { + self.has_stored_holes = false; + let db = self.region().db(); + let holes_name = self.holes_region_name(); + debug!("{}: removing holes region '{}'", db, holes_name); + db.remove_region(&holes_name)?; + } + + Ok(true) + } + + fn region(&self) -> &Region { + self.base.region() + } + + fn serialize_changes(&self) -> Result> { + self.serialize_raw_changes() + } + + fn any_stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + >::stamped_write_with_changes(self, stamp) + } + + fn remove(self) -> Result<()> { + Self::remove(self) + } + + fn any_truncate_if_needed_at(&mut self, index: usize) -> Result<()> { + >::truncate_if_needed_at(self, index) + } + + fn any_reset(&mut self) -> Result<()> { + >::reset(self) + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/any_vec.rs b/crates/vecdb/src/variants/raw/inner/read_write/any_vec.rs new file mode 100644 index 000000000..4927a2268 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/any_vec.rs @@ -0,0 +1,49 @@ +use crate::{AnyVec, VecIndex, VecValue, Version, short_type_name}; + +use super::{super::RawStrategy, ReadWriteRawVec}; + +impl AnyVec for ReadWriteRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + #[inline] + fn version(&self) -> Version { + self.base.version() + } + + #[inline] + fn name(&self) -> &str { + self.base.name() + } + + #[inline] + fn len(&self) -> usize { + self.base.len() + } + + #[inline] + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } + + #[inline] + fn value_type_to_string(&self) -> &'static str { + short_type_name::() + } + + #[inline] + fn region_names(&self) -> Vec { + let mut names = vec![self.index_to_name()]; + if self.has_stored_holes { + names.push(self.holes_region_name()); + } + names + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/change.rs b/crates/vecdb/src/variants/raw/inner/read_write/change.rs new file mode 100644 index 000000000..1cda4e007 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/change.rs @@ -0,0 +1,9 @@ +use std::collections::BTreeSet; + +use crate::ChangeData; + +pub(super) struct RawChangeData { + pub base: ChangeData, + pub modifications: Vec<(usize, T)>, + pub prev_holes: BTreeSet, +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/mod.rs b/crates/vecdb/src/variants/raw/inner/read_write/mod.rs new file mode 100644 index 000000000..3ff0b7de2 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/mod.rs @@ -0,0 +1,594 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + marker::PhantomData, +}; + +use log::info; +use rawdb::{Reader, unlikely}; + +mod any_stored_vec; +mod any_vec; +mod change; +mod readable; +mod rollback; +mod typed; +mod writable; + +use crate::{ + AnyStoredVec, AnyVec, Bytes, Error, Format, HEADER_OFFSET, ImportOptions, MMAP_CROSSOVER_BYTES, + RawIoSource, RawMmapSource, ReadWriteBaseVec, Result, VecIndex, VecReader, VecValue, Version, + WithPrev, vec_region_name_with, +}; + +use super::{RawStrategy, ReadOnlyRawVec}; + +const VERSION: Version = Version::ONE; + +/// Core implementation for raw storage vectors shared by BytesVec and ZeroCopyVec. +/// +/// Parameterized by serialization strategy `S` to support different serialization approaches: +/// - `BytesStrategy`: Explicit little-endian serialization (portable) +/// - `ZeroCopyStrategy`: Native byte order via zerocopy (fast but not portable) +/// +/// Provides holes (deleted indices) and updated values tracking for both vec types. +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct ReadWriteRawVec { + pub(crate) base: ReadWriteBaseVec, + pub(super) holes: WithPrev>, + pub(super) updated: WithPrev>, + pub(super) has_stored_holes: bool, + _strategy: PhantomData, +} + +impl ReadWriteRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + pub const SIZE_OF_T: usize = size_of::(); + + pub fn read_only_clone(&self) -> ReadOnlyRawVec { + ReadOnlyRawVec::new(self.base.read_only_base()) + } + + /// # Warning + /// + /// This will DELETE all existing data on format/version errors. Use with caution. + pub fn forced_import_with(mut options: ImportOptions, format: Format) -> Result { + options.version = options.version + VERSION; + let res = Self::import_with(options, format); + match res { + Err(Error::WrongEndian) + | Err(Error::WrongLength { .. }) + | Err(Error::DifferentFormat { .. }) + | Err(Error::DifferentVersion { .. }) => { + info!("Resetting {}...", options.name); + options + .db + .remove_region_if_exists(&vec_region_name_with::(options.name))?; + Self::import_with(options, format) + } + _ => res, + } + } + + pub fn import_with(mut options: ImportOptions, format: Format) -> Result { + options.version = options.version + VERSION; + + let db = options.db; + let name = options.name; + + let base = ReadWriteBaseVec::import(options, format)?; + + // Raw format requires data to be aligned to SIZE_OF_T + let region_len = base.region().meta().len(); + if region_len > HEADER_OFFSET + && !(region_len - HEADER_OFFSET).is_multiple_of(Self::SIZE_OF_T) + { + return Err(Error::CorruptedRegion { + name: name.to_string(), + region_len, + }); + } + + let holes = db + .get_region(&Self::holes_region_name_with(name)) + .map(|region| { + region + .create_reader() + .read_all() + .chunks(size_of::()) + .map(usize::from_bytes) + .collect::>>() + }) + .transpose()?; + + let mut this = Self { + base, + has_stored_holes: holes.is_some(), + holes: WithPrev::new(holes.unwrap_or_default()), + updated: WithPrev::default(), + _strategy: PhantomData, + }; + + let len = this.real_stored_len(); + *this.base.mut_prev_stored_len() = len; + this.base.update_stored_len(len); + + Ok(this) + } + + pub fn remove(self) -> Result<()> { + let db = self.base.db(); + let holes_region_name = self.holes_region_name(); + let has_stored_holes = self.has_stored_holes; + + self.base.remove()?; + + if has_stored_holes { + db.remove_region(&holes_region_name)?; + } + + Ok(()) + } + + pub(super) fn holes_region_name(&self) -> String { + Self::holes_region_name_with(self.name()) + } + + fn holes_region_name_with(name: &str) -> String { + format!("{}_holes", vec_region_name_with::(name)) + } + + #[inline(always)] + pub fn holes(&self) -> &BTreeSet { + self.holes.current() + } + + #[inline(always)] + pub fn prev_holes(&self) -> &BTreeSet { + self.holes.previous() + } + + #[inline(always)] + pub fn mut_holes(&mut self) -> &mut BTreeSet { + self.holes.current_mut() + } + + #[inline(always)] + pub fn updated(&self) -> &BTreeMap { + self.updated.current() + } + + #[inline(always)] + pub fn mut_updated(&mut self) -> &mut BTreeMap { + self.updated.current_mut() + } + + #[inline(always)] + pub fn prev_updated(&self) -> &BTreeMap { + self.updated.previous() + } + + #[inline(always)] + pub fn pushed(&self) -> &[T] { + self.base.pushed() + } + + #[inline(always)] + pub fn mut_pushed(&mut self) -> &mut Vec { + self.base.mut_pushed() + } + + #[inline] + pub fn push(&mut self, value: T) { + self.base.mut_pushed().push(value); + } + + #[inline] + pub fn reserve_pushed(&mut self, additional: usize) { + self.base.reserve_pushed(additional); + } + + #[inline] + fn raw_reader(&self) -> Reader { + self.base.region().create_reader() + } + + #[inline] + pub fn reader(&self) -> VecReader { + VecReader::from_read_write(self) + } + + #[inline] + pub fn index_to_name(&self) -> String { + self.base.index_to_name() + } + + #[inline(always)] + fn unchecked_read_at(&self, index: usize, reader: &Reader) -> T { + let ptr = reader.prefixed(HEADER_OFFSET).as_ptr(); + unsafe { S::read_from_ptr(ptr, index * Self::SIZE_OF_T) } + } + + #[inline] + pub fn read_at_once(&self, index: usize) -> Result { + let len = self.stored_len(); + if index >= len { + return Err(Error::IndexTooHigh { + index, + len, + name: self.name().to_string(), + }); + } + + Ok(self.base.region().with_read_bytes(|bytes| unsafe { + S::read_from_ptr(bytes.as_ptr().add(HEADER_OFFSET), index * Self::SIZE_OF_T) + })) + } + + #[inline] + pub fn read_once(&self, index: I) -> Result { + self.read_at_once(index.to_usize()) + } + + /// Reads from an append-only vector's persisted or pushed layer. + /// + /// Use [`Self::get_with_reader`] when the vector may contain holes or updates. + #[inline(always)] + pub fn get_append_only(&self, index: I, reader: &VecReader) -> Option { + self.get_append_only_at(index.to_usize(), reader) + } + + /// Raw-index form of [`Self::get_append_only`]. + #[inline(always)] + pub fn get_append_only_at(&self, index: usize, reader: &VecReader) -> Option { + debug_assert!( + self.holes().is_empty() && self.updated().is_empty(), + "get_append_only requires a vector without holes or updates" + ); + // The reader snapshots the persisted boundary when it is created. + // Using that boundary avoids reloading SharedLen for every lookup and + // lets the inlined reader.get_at() reuse the same bounds check. + let stored_len = reader.len(); + debug_assert_eq!(stored_len, self.stored_len(), "stale VecReader"); + if index >= stored_len { + return self.base.pushed().get(index - stored_len).cloned(); + } + Some(reader.get_at(index)) + } + + /// Reads the current value across holes, updates, persisted data, and pushes. + #[inline] + pub fn get_with_reader(&self, index: I, reader: &VecReader) -> Option { + self.get_with_reader_at(index.to_usize(), reader) + } + + /// Raw-index form of [`Self::get_with_reader`]. + #[inline] + pub fn get_with_reader_at(&self, index: usize, reader: &VecReader) -> Option { + if unlikely(!self.holes().is_empty()) && self.holes().contains(&index) { + return None; + } + + let stored_len = reader.len(); + debug_assert_eq!(stored_len, self.stored_len(), "stale VecReader"); + + if index >= stored_len { + return self.base.pushed().get(index - stored_len).cloned(); + } + + if unlikely(!self.updated().is_empty()) + && let Some(updated_value) = self.updated().get(&index) + { + return Some(updated_value.clone()); + } + + Some(reader.get_at(index)) + } + + pub fn collect_holed(&self) -> Vec> { + self.collect_holed_range(0, self.len()) + } + + pub fn collect_holed_range(&self, from: usize, to: usize) -> Vec> { + let len = self.len(); + let from = from.min(len); + let to = to.min(len); + + if from >= to { + return vec![]; + } + + let reader = self.reader(); + + (from..to) + .map(|i| self.get_with_reader_at(i, &reader)) + .collect() + } + + #[inline] + pub fn update(&mut self, index: I, value: T) -> Result<()> { + self.update_at(index.to_usize(), value) + } + + #[inline] + pub fn update_at(&mut self, index: usize, value: T) -> Result<()> { + let stored_len = self.stored_len(); + + if index >= stored_len { + let Some(slot) = self.base.mut_pushed().get_mut(index - stored_len) else { + return Err(Error::IndexTooHigh { + index, + len: stored_len, + name: self.name().to_string(), + }); + }; + *slot = value; + return Ok(()); + } + + if !self.holes().is_empty() { + self.mut_holes().remove(&index); + } + + self.mut_updated().insert(index, value); + + Ok(()) + } + + #[inline] + pub fn delete(&mut self, index: I) { + self.delete_at(index.to_usize()) + } + + #[inline] + pub fn delete_at(&mut self, index: usize) { + if index < self.len() { + self.unchecked_delete_at(index); + } + } + + #[inline] + #[doc(hidden)] + pub fn unchecked_delete(&mut self, index: I) { + self.unchecked_delete_at(index.to_usize()) + } + + #[inline] + #[doc(hidden)] + pub fn unchecked_delete_at(&mut self, index: usize) { + if !self.updated().is_empty() { + self.mut_updated().remove(&index); + } + self.mut_holes().insert(index); + } + + #[inline] + pub fn get_first_empty_index(&self) -> I { + self.holes() + .first() + .copied() + .unwrap_or_else(|| self.base.len()) + .into() + } + + #[inline] + pub fn fill_first_hole_or_push(&mut self, value: T) -> Result { + if let Some(hole) = self.mut_holes().pop_first().map(I::from) { + self.update(hole, value)?; + return Ok(hole); + } + self.base.mut_pushed().push(value); + Ok(I::from(self.len() - 1)) + } + + pub fn take(&mut self, index: I, reader: &VecReader) -> Option { + self.take_at(index.to_usize(), reader) + } + + pub fn take_at(&mut self, index: usize, reader: &VecReader) -> Option { + let opt = self.get_with_reader_at(index, reader); + if opt.is_some() { + self.unchecked_delete_at(index); + } + opt + } + + pub(crate) fn collect_stored_range(&self, from: usize, to: usize) -> Result> { + let reader = self.raw_reader(); + Ok((from..to) + .map(|i| { + if let Some(val) = self.prev_updated().get(&i) { + val.clone() + } else { + self.unchecked_read_at(i, &reader) + } + }) + .collect()) + } + + #[inline] + pub(super) fn has_dirty_stored(&self) -> bool { + !self.holes().is_empty() || !self.updated().is_empty() + } + + pub(super) fn truncate_dirty_at(&mut self, index: usize) { + if self.holes().last().is_some_and(|&h| h >= index) { + self.mut_holes().split_off(&index); + } + if self + .updated() + .last_key_value() + .is_some_and(|(&k, _)| k >= index) + { + self.mut_updated().split_off(&index); + } + } + + #[inline(always)] + pub(super) fn fold_source B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + let range_bytes = (to - from) * Self::SIZE_OF_T; + if range_bytes > MMAP_CROSSOVER_BYTES { + RawIoSource::new(self, from, to).fold(init, f) + } else { + RawMmapSource::new(self, from, to).fold(init, f) + } + } + + #[inline(always)] + pub(super) fn try_fold_source std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result { + let range_bytes = (to - from) * Self::SIZE_OF_T; + if range_bytes > MMAP_CROSSOVER_BYTES { + RawIoSource::new(self, from, to).try_fold(init, f) + } else { + RawMmapSource::new(self, from, to).try_fold(init, f) + } + } + + /// Own implementation (not delegating to try_fold_dirty) so LLVM can vectorize without `?` penalty. + pub(super) fn fold_dirty B>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> B { + let stored_len = self.stored_len(); + let reader = self.raw_reader(); + let data_ptr = reader.prefixed(HEADER_OFFSET).as_ptr(); + let mut acc = init; + + let stored_to = to.min(stored_len); + let mut hole_iter = self.holes().range(from..to).peekable(); + let mut update_iter = self.updated().range(from..stored_to).peekable(); + + let mut byte_off = from * Self::SIZE_OF_T; + for i in from..stored_to { + if unlikely(hole_iter.peek() == Some(&&i)) { + hole_iter.next(); + byte_off += Self::SIZE_OF_T; + continue; + } + let val = if unlikely(update_iter.peek().is_some_and(|&(&k, _)| k == i)) { + update_iter.next().unwrap().1.clone() + } else { + unsafe { S::read_from_ptr(data_ptr, byte_off) } + }; + byte_off += Self::SIZE_OF_T; + acc = f(acc, val); + } + + let push_from = from.max(stored_len); + if push_from < to { + let pushed = self.base.pushed(); + for i in push_from..to { + if unlikely(hole_iter.peek() == Some(&&i)) { + hole_iter.next(); + continue; + } + if let Some(v) = pushed.get(i - stored_len) { + acc = f(acc, v.clone()); + } + } + } + + acc + } + + pub(super) fn try_fold_dirty std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result { + let stored_len = self.stored_len(); + let reader = self.raw_reader(); + let data_ptr = reader.prefixed(HEADER_OFFSET).as_ptr(); + let mut acc = init; + + let stored_to = to.min(stored_len); + let mut hole_iter = self.holes().range(from..to).peekable(); + let mut update_iter = self.updated().range(from..stored_to).peekable(); + + let mut byte_off = from * Self::SIZE_OF_T; + for i in from..stored_to { + if unlikely(hole_iter.peek() == Some(&&i)) { + hole_iter.next(); + byte_off += Self::SIZE_OF_T; + continue; + } + let val = if unlikely(update_iter.peek().is_some_and(|&(&k, _)| k == i)) { + update_iter.next().unwrap().1.clone() + } else { + // SAFETY: i < stored_len, reader holds mmap guard + unsafe { S::read_from_ptr(data_ptr, byte_off) } + }; + byte_off += Self::SIZE_OF_T; + acc = f(acc, val)?; + } + + let push_from = from.max(stored_len); + if push_from < to { + let pushed = self.base.pushed(); + for i in push_from..to { + if unlikely(hole_iter.peek() == Some(&&i)) { + hole_iter.next(); + continue; + } + if let Some(v) = pushed.get(i - stored_len) { + acc = f(acc, v.clone())?; + } + } + } + + Ok(acc) + } + + pub fn fold_stored_io B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + let stored_len = self.stored_len(); + let from = from.min(stored_len); + let to = to.min(stored_len); + if from >= to { + return init; + } + RawIoSource::new(self, from, to).fold(init, f) + } + + pub fn fold_stored_mmap B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + let stored_len = self.stored_len(); + let from = from.min(stored_len); + let to = to.min(stored_len); + if from >= to { + return init; + } + RawMmapSource::new(self, from, to).fold(init, f) + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/readable.rs b/crates/vecdb/src/variants/raw/inner/read_write/readable.rs new file mode 100644 index 000000000..d1fa95ed5 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/readable.rs @@ -0,0 +1,155 @@ +use rawdb::unlikely; + +use crate::{AnyStoredVec, HEADER_OFFSET, ReadableVec, VecIndex, VecValue}; + +use super::{super::RawStrategy, ReadWriteRawVec}; + +impl ReadableVec for ReadWriteRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + #[inline(always)] + fn collect_one_at(&self, index: usize) -> Option { + let len = self.base.len(); + if index >= len { + return None; + } + + if unlikely(!self.holes().is_empty()) && self.holes().contains(&index) { + return None; + } + + let stored_len = self.stored_len(); + if index >= stored_len { + return self.base.pushed().get(index - stored_len).cloned(); + } + + if unlikely(!self.updated().is_empty()) + && let Some(value) = self.updated().get(&index) + { + return Some(value.clone()); + } + + Some(self.base.region().with_read_bytes(|bytes| unsafe { + S::read_from_ptr(bytes.as_ptr().add(HEADER_OFFSET), index * size_of::()) + })) + } + + #[inline(always)] + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec) { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return; + } + + buf.reserve(to - from); + + if self.has_dirty_stored() { + self.fold_dirty(from, to, (), |(), v| buf.push(v)); + return; + } + + let stored_len = self.stored_len(); + + if from < stored_len { + let stored_to = to.min(stored_len); + if S::IS_NATIVE_LAYOUT { + // Bulk read: memory layout matches T, single memcpy from mmap. + let reader = self.raw_reader(); + let src = unsafe { + std::slice::from_raw_parts( + reader + .prefixed(HEADER_OFFSET) + .as_ptr() + .add(from * Self::SIZE_OF_T) as *const T, + stored_to - from, + ) + }; + buf.extend_from_slice(src); + } else { + self.fold_source(from, stored_to, (), |(), v| buf.push(v)); + } + } + + if to > stored_len { + let push_from = from.max(stored_len); + let pushed = self.base.pushed(); + let start = push_from - stored_len; + let end = (to - stored_len).min(pushed.len()); + buf.extend_from_slice(&pushed[start..end]); + } + } + + #[inline] + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(T)) { + self.fold_range_at(from, to, (), |(), v| f(v)); + } + + #[inline] + fn fold_range_at B>(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return init; + } + + if self.has_dirty_stored() { + return self.fold_dirty(from, to, init, f); + } + + let stored_len = self.stored_len(); + + if to <= stored_len { + return self.fold_source(from, to, init, f); + } + + let mut acc = init; + if from < stored_len { + acc = self.fold_source(from, stored_len, acc, &mut f); + } + self.base.fold_pushed(from, to, acc, f) + } + + #[inline] + fn try_fold_range_at std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + mut f: F, + ) -> std::result::Result + where + Self: Sized, + { + let len = self.base.len(); + let from = from.min(len); + let to = to.min(len); + if from >= to { + return Ok(init); + } + + if self.has_dirty_stored() { + return self.try_fold_dirty(from, to, init, f); + } + + let stored_len = self.stored_len(); + + if to <= stored_len { + return self.try_fold_source(from, to, init, f); + } + + let mut acc = init; + if from < stored_len { + acc = self.try_fold_source(from, stored_len, acc, &mut f)?; + } + self.base.try_fold_pushed(from, to, acc, f) + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/rollback.rs b/crates/vecdb/src/variants/raw/inner/read_write/rollback.rs new file mode 100644 index 000000000..448bd3a67 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/rollback.rs @@ -0,0 +1,123 @@ +use std::collections::BTreeSet; + +use crate::{ + AnyStoredVec, Bytes, ChangeCursor, ChangeData, ReadWriteBaseVec, Result, SIZE_OF_U64, VecIndex, + VecValue, +}; + +use super::{super::RawStrategy, ReadWriteRawVec, change::RawChangeData}; + +impl ReadWriteRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + pub(super) fn serialize_raw_changes(&self) -> Result> { + let mut bytes = self.base.serialize_changes( + Self::SIZE_OF_T, + |from, to| self.collect_stored_range(from, to), + |vals, buf| { + for v in vals { + S::write_to_vec(v, buf); + } + }, + )?; + + let reader = self.raw_reader(); + let updated = self.updated(); + let prev_updated = self.prev_updated(); + + // Collect all indices that need change tracking: entries currently modified + // AND entries that were in prev_updated but removed (e.g., by delete_at). + // Without the latter, rollback after rollback loses track of deleted entries. + let all_keys: BTreeSet = + updated.keys().chain(prev_updated.keys()).copied().collect(); + + bytes.extend(all_keys.len().to_bytes()); + for &i in &all_keys { + bytes.extend(i.to_bytes()); + } + for &i in &all_keys { + if let Some(v) = prev_updated.get(&i) { + S::write_to_vec(v, &mut bytes); + } else { + S::write_to_vec(&self.unchecked_read_at(i, &reader), &mut bytes); + } + } + + let prev_holes = self.prev_holes(); + bytes.extend(prev_holes.len().to_bytes()); + for &hole in prev_holes { + bytes.extend(hole.to_bytes()); + } + + Ok(bytes) + } + + fn parse_raw_change_data(bytes: &[u8]) -> Result> { + let mut c = ChangeCursor::new(bytes); + let base = + ReadWriteBaseVec::::parse_change_data(&mut c, Self::SIZE_OF_T, |b| S::read(b))?; + + let modified_len = c.read_u64()?; + let indices = c.read_values(modified_len, SIZE_OF_U64, usize::from_bytes)?; + let values = c.read_values(modified_len, Self::SIZE_OF_T, |b| S::read(b))?; + let modifications = indices.into_iter().zip(values).collect(); + + let prev_holes_len = c.read_u64()?; + let prev_holes = c + .read_values(prev_holes_len, SIZE_OF_U64, usize::from_bytes)? + .into_iter() + .collect(); + + Ok(RawChangeData { + base, + modifications, + prev_holes, + }) + } + + pub(super) fn deserialize_then_undo_changes(&mut self, bytes: &[u8]) -> Result<()> { + let RawChangeData { + base: + ChangeData { + prev_stamp, + prev_stored_len, + truncated_start, + truncated_values, + prev_pushed, + .. + }, + modifications, + prev_holes, + } = Self::parse_raw_change_data(bytes)?; + + // Only needed when the rolled-back flush appended (prev_stored_len < + // current): any holes/updated in the now-gone range must be dropped. + if prev_stored_len < self.stored_len() { + self.truncate_dirty_at(prev_stored_len); + } + + // Truncated values overlay via `updated` at indices beyond the + // current on-disk length; the next write() extends the region. + self.base + .apply_rollback(prev_stamp, prev_stored_len, prev_pushed); + for (i, val) in truncated_values.into_iter().enumerate() { + self.mut_updated().insert(truncated_start + i, val); + } + + for (idx, val) in modifications { + self.update_at(idx, val)?; + } + + if !prev_holes.is_empty() || !self.holes().is_empty() || !self.prev_holes().is_empty() { + *self.holes.current_mut() = prev_holes; + self.holes.save(); + } + + self.updated.save(); + + Ok(()) + } +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/typed.rs b/crates/vecdb/src/variants/raw/inner/read_write/typed.rs new file mode 100644 index 000000000..2c58f1320 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/typed.rs @@ -0,0 +1,13 @@ +use crate::{TypedVec, VecIndex, VecValue}; + +use super::{super::RawStrategy, ReadWriteRawVec}; + +impl TypedVec for ReadWriteRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + type I = I; + type T = T; +} diff --git a/crates/vecdb/src/variants/raw/inner/read_write/writable.rs b/crates/vecdb/src/variants/raw/inner/read_write/writable.rs new file mode 100644 index 000000000..cc3e25e64 --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/read_write/writable.rs @@ -0,0 +1,80 @@ +use std::{collections::BTreeMap, path::PathBuf}; + +use crate::{AnyStoredVec, Result, Stamp, VecIndex, VecValue, WritableVec}; + +use super::{super::RawStrategy, ReadWriteRawVec}; + +impl WritableVec for ReadWriteRawVec +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + #[inline] + fn push(&mut self, value: T) { + self.base.mut_pushed().push(value); + } + + #[inline] + fn pushed(&self) -> &[T] { + self.base.pushed() + } + + fn truncate_if_needed_at(&mut self, index: usize) -> Result<()> { + self.truncate_dirty_at(index); + + if self.base.truncate_pushed(index) { + self.base.update_stored_len(index); + } + + Ok(()) + } + + fn reset(&mut self) -> Result<()> { + self.holes.clear(); + self.updated.clear(); + self.truncate_if_needed_at(0)?; + self.base.reset_base() + } + + fn reset_unsaved(&mut self) { + self.base.reset_unsaved_base(); + self.holes.clear(); + self.updated.clear(); + } + + fn is_dirty(&self) -> bool { + !self.base.pushed().is_empty() || !self.holes().is_empty() || !self.updated().is_empty() + } + + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + if self.base.saved_stamped_changes() == 0 { + return self.stamped_write(stamp); + } + + // serialize_changes() reads prev_holes, so must happen BEFORE holes.save() + let data = self.serialize_changes()?; + self.base.save_change_file(stamp, &data)?; + self.stamped_write(stamp)?; + self.base.save_prev(); + self.holes.save(); + self.updated.clear_previous(); + + Ok(()) + } + + fn rollback(&mut self) -> Result<()> { + let bytes = self.base.read_current_change_file()?; + self.deserialize_then_undo_changes(&bytes) + } + + fn find_rollback_files(&self) -> Result> { + self.base.find_rollback_files() + } + + fn save_rollback_state(&mut self) { + self.base.save_prev_for_rollback(); + self.holes.save(); + self.updated.save(); + } +} diff --git a/crates/vecdb/src/variants/raw/inner/strategy.rs b/crates/vecdb/src/variants/raw/inner/strategy.rs new file mode 100644 index 000000000..c62f1115e --- /dev/null +++ b/crates/vecdb/src/variants/raw/inner/strategy.rs @@ -0,0 +1,16 @@ +use crate::ValueStrategy; + +/// Strategy for raw (uncompressed) storage vectors. +/// +/// Extends `ValueStrategy` with fast pointer-based reads used by mmap/IO iterators. +pub trait RawStrategy: ValueStrategy { + /// Reads a single T from a raw byte pointer at the given byte offset. + /// + /// For native-layout types, this compiles to a single `mov` instruction, + /// bypassing slice creation, bounds checking, and Result overhead. + /// + /// # Safety + /// - `ptr.add(byte_offset)` must be valid for reading `size_of::()` bytes. + /// - The bytes at that location must be a valid serialized T. + unsafe fn read_from_ptr(ptr: *const u8, byte_offset: usize) -> T; +} diff --git a/crates/vecdb/src/variants/raw/mod.rs b/crates/vecdb/src/variants/raw/mod.rs new file mode 100644 index 000000000..bfaa2a123 --- /dev/null +++ b/crates/vecdb/src/variants/raw/mod.rs @@ -0,0 +1,12 @@ +mod bytes; +mod inner; +mod sources; +#[cfg(feature = "zerocopy")] +mod zerocopy; + +pub use bytes::*; +pub use inner::*; +pub(crate) use sources::*; +pub use sources::{VecReader, VecReaderCursor}; +#[cfg(feature = "zerocopy")] +pub use zerocopy::*; diff --git a/crates/vecdb/src/variants/raw/sources/io.rs b/crates/vecdb/src/variants/raw/sources/io.rs new file mode 100644 index 000000000..a56e3f495 --- /dev/null +++ b/crates/vecdb/src/variants/raw/sources/io.rs @@ -0,0 +1,157 @@ +use std::{ + fs::File, + io::{Read, Seek, SeekFrom}, + marker::PhantomData, +}; + +use parking_lot::RwLockReadGuard; +use rawdb::{Region, RegionMetadata}; + +use crate::{AnyStoredVec, BUFFER_SIZE, HEADER_OFFSET, VecIndex, VecValue, likely}; + +use super::super::{RawStrategy, ReadWriteRawVec}; + +/// Buffer size aligned to SIZE_OF_T for raw I/O reads. +const fn aligned_buffer_size() -> usize { + let size_of_t = size_of::(); + (BUFFER_SIZE / size_of_t) * size_of_t +} + +/// Buffered file I/O source for reading stored data sequentially. +/// +/// Better than mmap for very large sequential scans (>4 GiB). Uses a dedicated +/// file handle with OS readahead. Only sees stored (persisted) values. +pub struct RawIoSource<'a, I, T, S> { + file: File, + buffer: Vec, + buffer_pos: usize, + buffer_len: usize, + file_offset: usize, + end_offset: usize, + _lock: RwLockReadGuard<'a, RegionMetadata>, + _marker: PhantomData<(I, T, S)>, +} + +impl<'a, I, T, S> RawIoSource<'a, I, T, S> +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + const SIZE_OF_T: usize = size_of::(); + const NORMAL_BUFFER_SIZE: usize = aligned_buffer_size::(); + + pub(crate) fn new(vec: &'a ReadWriteRawVec, from: usize, to: usize) -> Self { + Self::new_from_parts(vec.region(), vec.stored_len(), from, to) + } + + pub(crate) fn new_from_parts( + region: &'a Region, + stored_len: usize, + from: usize, + to: usize, + ) -> Self { + let file = region.open_db_read_only_file().expect("open file"); + let region_meta = region.meta(); + let region_start = region_meta.start(); + let start_offset = region_start + HEADER_OFFSET; + let from = from.min(stored_len); + let to = to.min(stored_len); + + let from_offset = start_offset + from * Self::SIZE_OF_T; + let end_offset = start_offset + to * Self::SIZE_OF_T; + + let mut this = Self { + file, + buffer: vec![0; Self::NORMAL_BUFFER_SIZE], + buffer_pos: 0, + buffer_len: 0, + file_offset: from_offset, + end_offset, + _lock: region_meta, + _marker: PhantomData, + }; + + if likely(this.can_read_file()) { + this.file + .seek(SeekFrom::Start(from_offset as u64)) + .expect("Failed to seek"); + } + + this + } + + #[inline(always)] + fn can_read_file(&self) -> bool { + self.file_offset < self.end_offset + } + + #[inline(always)] + fn cant_read_file(&self) -> bool { + self.file_offset >= self.end_offset + } + + #[inline(always)] + fn remaining_file_bytes(&self) -> usize { + self.end_offset - self.file_offset + } + + #[inline(always)] + fn refill_buffer(&mut self) { + let buffer_len = self.remaining_file_bytes().min(Self::NORMAL_BUFFER_SIZE); + self.file + .read_exact(&mut self.buffer[..buffer_len]) + .expect("Failed to read file buffer"); + self.file_offset += buffer_len; + self.buffer_len = buffer_len; + self.buffer_pos = 0; + } + + /// Fold all remaining elements — own implementation so LLVM can vectorize the inner loop. + #[inline(always)] + pub(crate) fn fold B>(mut self, init: B, mut f: F) -> B { + let mut accum = init; + loop { + let ptr = self.buffer.as_ptr(); + let mut pos = self.buffer_pos; + let end = self.buffer_len; + while pos + Self::SIZE_OF_T <= end { + accum = f(accum, unsafe { S::read_from_ptr(ptr, pos) }); + pos += Self::SIZE_OF_T; + } + self.buffer_pos = end; + + if self.cant_read_file() { + break; + } + self.refill_buffer(); + } + accum + } + + /// Fallible fold with early exit on error. + #[inline(always)] + pub(crate) fn try_fold std::result::Result>( + mut self, + init: B, + mut f: F, + ) -> std::result::Result { + let mut accum = init; + loop { + let ptr = self.buffer.as_ptr(); + let mut pos = self.buffer_pos; + let end = self.buffer_len; + while pos + Self::SIZE_OF_T <= end { + accum = f(accum, unsafe { S::read_from_ptr(ptr, pos) })?; + pos += Self::SIZE_OF_T; + } + self.buffer_pos = end; + + if self.cant_read_file() { + break; + } + self.refill_buffer(); + } + Ok(accum) + } +} diff --git a/crates/vecdb/src/variants/raw/sources/mmap.rs b/crates/vecdb/src/variants/raw/sources/mmap.rs new file mode 100644 index 000000000..5893652c5 --- /dev/null +++ b/crates/vecdb/src/variants/raw/sources/mmap.rs @@ -0,0 +1,98 @@ +use std::marker::PhantomData; + +use rawdb::Reader; + +use rawdb::Region; + +use crate::{AnyStoredVec, HEADER_OFFSET, VecIndex, VecValue}; + +use super::super::{RawStrategy, ReadWriteRawVec}; + +/// Read-only mmap-backed source over a raw (uncompressed) vector. +/// +/// Only sees **stored** (persisted) values — pushed but unflushed values +/// are not visible. Created with a range and consumed by fold/try_fold/for_each. +/// +/// The data slice is computed once at construction time (matching Reader's +/// own `transmute` pattern), so fold/for_each are direct slice operations. +pub struct RawMmapSource { + // SAFETY: Field order matters. `_reader` keeps the mmap guard alive. + // `data` is a pointer into that mmap. `_reader` must outlive `data`, + // which it does because `data` is a raw pointer with no destructor. + _reader: Reader, + data: *const u8, + pos: usize, + end: usize, + _marker: PhantomData<(I, T, S)>, +} + +// SAFETY: RawMmapSource is read-only. The mmap data it points to is shared +// immutable memory protected by Reader's RwLockReadGuard, which is Sync. +unsafe impl Send for RawMmapSource {} +unsafe impl Sync for RawMmapSource {} + +impl RawMmapSource +where + I: VecIndex, + T: VecValue, + S: RawStrategy, +{ + const SIZE_OF_T: usize = size_of::(); + + pub(crate) fn new(vec: &ReadWriteRawVec, from: usize, to: usize) -> Self { + Self::new_from_parts(vec.region(), vec.stored_len(), from, to) + } + + pub(crate) fn new_from_parts( + region: &Region, + stored_len: usize, + from: usize, + to: usize, + ) -> Self { + let reader = region.create_reader(); + let from = from.min(stored_len); + let to = to.min(stored_len); + let slice = reader.prefixed(HEADER_OFFSET); + let ptr = slice.as_ptr(); + + Self { + _reader: reader, + data: ptr, + pos: from, + end: to, + _marker: PhantomData, + } + } + + /// Fold all elements in the range — tight pointer loop. + #[inline(always)] + pub(crate) fn fold B>(self, init: B, mut f: F) -> B { + let ptr = self.data; + let mut byte_off = self.pos * Self::SIZE_OF_T; + let end_byte = self.end * Self::SIZE_OF_T; + let mut acc = init; + while byte_off < end_byte { + acc = f(acc, unsafe { S::read_from_ptr(ptr, byte_off) }); + byte_off += Self::SIZE_OF_T; + } + acc + } + + /// Fallible fold with early exit on error. + #[inline(always)] + pub(crate) fn try_fold std::result::Result>( + self, + init: B, + mut f: F, + ) -> std::result::Result { + let ptr = self.data; + let mut byte_off = self.pos * Self::SIZE_OF_T; + let end_byte = self.end * Self::SIZE_OF_T; + let mut acc = init; + while byte_off < end_byte { + acc = f(acc, unsafe { S::read_from_ptr(ptr, byte_off) })?; + byte_off += Self::SIZE_OF_T; + } + Ok(acc) + } +} diff --git a/crates/vecdb/src/variants/raw/sources/mod.rs b/crates/vecdb/src/variants/raw/sources/mod.rs new file mode 100644 index 000000000..31d1ef744 --- /dev/null +++ b/crates/vecdb/src/variants/raw/sources/mod.rs @@ -0,0 +1,7 @@ +mod io; +mod mmap; +mod reader; + +pub(crate) use io::*; +pub(crate) use mmap::*; +pub use reader::*; diff --git a/crates/vecdb/src/variants/raw/sources/reader.rs b/crates/vecdb/src/variants/raw/sources/reader.rs new file mode 100644 index 000000000..e1077b335 --- /dev/null +++ b/crates/vecdb/src/variants/raw/sources/reader.rs @@ -0,0 +1,203 @@ +use std::marker::PhantomData; + +use rawdb::{Reader, Region}; + +use crate::{AnyStoredVec, HEADER_OFFSET, ReadOnlyRawVec, VecIndex, VecValue}; + +use super::super::{RawStrategy, ReadWriteRawVec}; + +/// Read-only random-access handle into a raw vector's stored data. +/// +/// Created via `raw_vec.reader()` (available on BytesVec/ZeroCopyVec via Deref). +/// Provides O(1) point reads directly from the memory-mapped file. +/// +/// Only sees **stored** (persisted) values — does not check holes, updates, +/// or pushed values. For full dirty-state reads, use `get_with_reader`. +/// +/// The reader holds the current mmap generation alive. Drop long-lived readers +/// before writes that may grow the database and remap the file. +pub struct VecReader { + _reader: Reader, + data: *const u8, + stored_len: usize, + _marker: PhantomData<(I, T, S)>, +} + +/// Forward cursor over a raw vector reader. +/// +/// Unlike [`crate::Cursor`], this reads values directly from the existing mmap +/// and does not allocate or copy through a staging buffer. Like [`VecReader`], +/// it only sees persisted values. +pub struct VecReaderCursor { + reader: VecReader, + pos: usize, +} + +unsafe impl Send for VecReader {} +unsafe impl Sync for VecReader {} + +impl VecReader +where + T: VecValue, + S: RawStrategy, +{ + const SIZE_OF_T: usize = size_of::(); + + pub(crate) fn from_region(region: &Region, stored_len: usize) -> Self { + let reader = region.create_reader(); + let slice = reader.prefixed(HEADER_OFFSET); + let ptr = slice.as_ptr(); + + Self { + _reader: reader, + data: ptr, + stored_len, + _marker: PhantomData, + } + } + + pub(crate) fn from_read_write(vec: &ReadWriteRawVec) -> Self + where + I: VecIndex, + { + Self::from_region(vec.region(), vec.stored_len()) + } + + pub(crate) fn from_read_only(vec: &ReadOnlyRawVec) -> Self + where + I: VecIndex, + { + Self::from_region(vec.region(), vec.stored_len()) + } + + /// Returns the value at typed `index`. + /// + /// # Panics + /// Panics if `index >= len()`. + #[inline(always)] + pub fn get(&self, index: I) -> T + where + I: VecIndex, + { + self.get_at(index.to_usize()) + } + + /// Returns the value at raw `index`. + /// + /// # Panics + /// Panics if `index >= len()`. + #[inline(always)] + pub fn get_at(&self, index: usize) -> T { + assert!( + index < self.stored_len, + "index {index} out of bounds (len {})", + self.stored_len + ); + // SAFETY: index < stored_len guarantees offset + SIZE_OF_T <= data_len + unsafe { S::read_from_ptr(self.data, index * Self::SIZE_OF_T) } + } + + /// Returns the value at typed `index`, or `None` if out of bounds. + #[inline(always)] + pub fn try_get(&self, index: I) -> Option + where + I: VecIndex, + { + self.try_get_at(index.to_usize()) + } + + /// Returns the value at raw `index`, or `None` if out of bounds. + #[inline(always)] + pub fn try_get_at(&self, index: usize) -> Option { + if index >= self.stored_len { + return None; + } + // SAFETY: index < stored_len guarantees offset + SIZE_OF_T <= data_len + Some(unsafe { S::read_from_ptr(self.data, index * Self::SIZE_OF_T) }) + } + + /// Returns the number of stored values. + #[inline(always)] + pub fn len(&self) -> usize { + self.stored_len + } + + /// Returns `true` if the reader is empty. + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.stored_len == 0 + } + + #[inline(always)] + #[cfg(feature = "zerocopy")] + pub(crate) fn as_bytes(&self) -> &[u8] { + // SAFETY: `data` points to `stored_len * SIZE_OF_T` bytes and `_reader` + // keeps the mmap generation containing them alive for this borrow. + unsafe { std::slice::from_raw_parts(self.data, self.stored_len * Self::SIZE_OF_T) } + } + + /// Creates an allocation-free cursor over the persisted values. + #[inline] + pub fn cursor(self) -> VecReaderCursor { + VecReaderCursor { + reader: self, + pos: 0, + } + } +} + +impl VecReaderCursor +where + T: VecValue, + S: RawStrategy, +{ + /// Returns the current absolute position. + #[inline(always)] + pub fn position(&self) -> usize { + self.pos + } + + /// Returns the number of values remaining. + #[inline(always)] + pub fn remaining(&self) -> usize { + self.reader.len().saturating_sub(self.pos) + } + + /// Advances the position by `n` without reading. + #[inline(always)] + pub fn advance(&mut self, n: usize) { + self.pos = self.pos.saturating_add(n).min(self.reader.len()); + } + + /// Returns the value at absolute `index` without changing the position. + #[inline(always)] + pub fn get(&self, index: usize) -> Option { + self.reader.try_get_at(index) + } + + /// Returns the next value and advances the position. + #[inline(always)] + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Option { + let value = self.reader.try_get_at(self.pos)?; + self.pos += 1; + Some(value) + } + + /// Folds over the next `n` values and advances the position. + #[inline] + pub fn fold(&mut self, n: usize, mut init: B, mut f: impl FnMut(B, T) -> B) -> B { + let end = self.pos.saturating_add(n).min(self.reader.len()); + while self.pos < end { + init = f(init, self.reader.get_at(self.pos)); + self.pos += 1; + } + init + } + + /// Calls `f` for each of the next `n` values and advances the position. + #[inline] + pub fn for_each(&mut self, n: usize, mut f: impl FnMut(T)) { + self.fold(n, (), |(), value| f(value)); + } +} diff --git a/crates/vecdb/src/variants/raw/zerocopy/mod.rs b/crates/vecdb/src/variants/raw/zerocopy/mod.rs new file mode 100644 index 000000000..5790e0bab --- /dev/null +++ b/crates/vecdb/src/variants/raw/zerocopy/mod.rs @@ -0,0 +1,112 @@ +use crate::{AnyStoredVec, Format, ReadOnlyRawVec, VecIndex, VecReader, impl_vec_wrapper}; + +use super::ReadWriteRawVec; + +mod strategy; +mod value; + +pub use strategy::*; +pub use value::*; + +/// Raw storage vector using zerocopy for direct memory mapping in native byte order. +/// +/// Uses the `zerocopy` crate for direct memory-mapped access without copying, providing +/// the fastest possible performance. Values are stored in **NATIVE byte order**. +/// +/// Like `BytesVec`, this wraps `ReadWriteRawVec` and supports: +/// - Holes (deleted indices) +/// - Updated values (modifications to stored data) +/// - Push/rollback operations +/// +/// The only difference from `BytesVec` is the serialization strategy: +/// - `ZeroCopyVec`: Native byte order, faster but not portable +/// - `BytesVec`: Explicit little-endian, portable across architectures +/// +/// # Portability Warning +/// +/// **NOT portable across systems with different endianness.** Data written on a +/// little-endian system (x86) cannot be read correctly on a big-endian system. +/// For portable storage, use `BytesVec` instead. +/// +/// Use `ZeroCopyVec` when: +/// - Maximum performance is critical +/// - Data stays on the same architecture +/// +/// Use `BytesVec` when: +/// - Cross-platform compatibility is needed +/// - Sharing data between different architectures +#[derive(Debug)] +#[must_use = "Vector should be stored to keep data accessible"] +pub struct ZeroCopyVec(pub(crate) ReadWriteRawVec>); + +impl ZeroCopyVec +where + I: VecIndex, + T: ZeroCopyVecValue, +{ + /// The size of T in bytes. + pub const SIZE_OF_T: usize = size_of::(); + + /// Returns a reference to the value directly from the memory-mapped file without copying. + /// Very efficient for large types or frequent reads. + /// + /// Returns `None` if: + /// - Index is marked as a hole (deleted) + /// - Index is beyond stored length (might be in pushed layer) + /// - Index has an updated value (in the updated map, not on disk) + #[inline] + pub fn read_ref<'a>( + &self, + index: I, + reader: &'a VecReader>, + ) -> Option<&'a T> { + self.read_ref_at(index.to_usize(), reader) + } + + /// Returns a reference to the value at the given usize index directly from the memory-mapped file. + #[inline] + pub fn read_ref_at<'a>( + &self, + index: usize, + reader: &'a VecReader>, + ) -> Option<&'a T> { + // Cannot return ref for holes + if !self.holes().is_empty() && self.holes().contains(&index) { + return None; + } + + let stored_len = reader.len(); + debug_assert_eq!(stored_len, self.stored_len(), "stale VecReader"); + + // Cannot return ref for pushed values (they're in a Vec, not mmap) + if index >= stored_len { + return None; + } + + // Cannot return ref for updated values (they're in a BTreeMap, not mmap) + if !self.updated().is_empty() && self.updated().contains_key(&index) { + return None; + } + + self.stored_ref_at(index, reader) + } + + #[inline] + fn stored_ref_at<'a>( + &self, + index: usize, + reader: &'a VecReader>, + ) -> Option<&'a T> { + let offset = index * Self::SIZE_OF_T; + let bytes = reader.as_bytes().get(offset..)?; + T::ref_from_prefix(bytes).map(|(v, _)| v).ok() + } +} + +impl_vec_wrapper!( + ZeroCopyVec, + ReadWriteRawVec>, + ZeroCopyVecValue, + Format::ZeroCopy, + ReadOnlyRawVec> +); diff --git a/crates/vecdb/src/variants/raw/zerocopy/strategy/mod.rs b/crates/vecdb/src/variants/raw/zerocopy/strategy/mod.rs new file mode 100644 index 000000000..44dbe5c23 --- /dev/null +++ b/crates/vecdb/src/variants/raw/zerocopy/strategy/mod.rs @@ -0,0 +1,10 @@ +use std::marker::PhantomData; + +mod raw; +mod value; + +/// Serialization strategy using zerocopy for native byte order access. +/// +/// Uses direct memory mapping in native byte order - not portable across endianness. +#[derive(Debug, Clone, Copy)] +pub struct ZeroCopyStrategy(PhantomData); diff --git a/crates/vecdb/src/variants/raw/zerocopy/strategy/raw.rs b/crates/vecdb/src/variants/raw/zerocopy/strategy/raw.rs new file mode 100644 index 000000000..96274e342 --- /dev/null +++ b/crates/vecdb/src/variants/raw/zerocopy/strategy/raw.rs @@ -0,0 +1,10 @@ +use crate::{ZeroCopyVecValue, variants::raw::RawStrategy}; + +use super::ZeroCopyStrategy; + +impl RawStrategy for ZeroCopyStrategy { + #[inline(always)] + unsafe fn read_from_ptr(ptr: *const u8, byte_offset: usize) -> T { + unsafe { (ptr.add(byte_offset) as *const T).read_unaligned() } + } +} diff --git a/crates/vecdb/src/variants/raw/zerocopy/strategy/value.rs b/crates/vecdb/src/variants/raw/zerocopy/strategy/value.rs new file mode 100644 index 000000000..ae8dc1f11 --- /dev/null +++ b/crates/vecdb/src/variants/raw/zerocopy/strategy/value.rs @@ -0,0 +1,24 @@ +use crate::{Error, Result, ValueStrategy, ZeroCopyVecValue}; + +use super::ZeroCopyStrategy; + +impl ValueStrategy for ZeroCopyStrategy { + const IS_NATIVE_LAYOUT: bool = true; + + #[inline(always)] + fn read(bytes: &[u8]) -> Result { + T::read_from_prefix(bytes) + .map(|(v, _)| v) + .map_err(|_| Error::ZeroCopyError) + } + + #[inline(always)] + fn write_to_vec(value: &T, buf: &mut Vec) { + buf.extend_from_slice(value.as_bytes()); + } + + #[inline(always)] + fn write_to_slice(value: &T, dst: &mut [u8]) { + dst.copy_from_slice(value.as_bytes()); + } +} diff --git a/crates/vecdb/src/variants/raw/zerocopy/value.rs b/crates/vecdb/src/variants/raw/zerocopy/value.rs new file mode 100644 index 000000000..27f2f6ddf --- /dev/null +++ b/crates/vecdb/src/variants/raw/zerocopy/value.rs @@ -0,0 +1,13 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::VecValue; + +/// Value trait for ZeroCopyVec. +/// Extends RawVecValue with zerocopy bounds for direct memory mapping. +pub trait ZeroCopyVecValue +where + Self: VecValue + FromBytes + IntoBytes + Immutable + KnownLayout, +{ +} + +impl ZeroCopyVecValue for T where T: VecValue + FromBytes + IntoBytes + Immutable + KnownLayout {} diff --git a/crates/vecdb/src/version/add.rs b/crates/vecdb/src/version/add.rs new file mode 100644 index 000000000..07bfce053 --- /dev/null +++ b/crates/vecdb/src/version/add.rs @@ -0,0 +1,10 @@ +use std::ops::Add; + +use super::Version; + +impl Add for Version { + type Output = Self; + fn add(self, rhs: Version) -> Self::Output { + Self(self.0 + rhs.0) + } +} diff --git a/crates/vecdb/src/version/bytes.rs b/crates/vecdb/src/version/bytes.rs new file mode 100644 index 000000000..0a807c109 --- /dev/null +++ b/crates/vecdb/src/version/bytes.rs @@ -0,0 +1,17 @@ +use crate::{Bytes, Result}; + +use super::Version; + +impl Bytes for Version { + type Array = [u8; size_of::()]; + + #[inline] + fn to_bytes(&self) -> Self::Array { + self.0.to_bytes() + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> Result { + Ok(Self(u32::from_bytes(bytes)?)) + } +} diff --git a/crates/vecdb/src/version/conversions.rs b/crates/vecdb/src/version/conversions.rs new file mode 100644 index 000000000..02a400d02 --- /dev/null +++ b/crates/vecdb/src/version/conversions.rs @@ -0,0 +1,26 @@ +use super::Version; + +impl From for u32 { + fn from(value: Version) -> u32 { + value.0 + } +} + +impl From for usize { + fn from(value: Version) -> usize { + value.0 as usize + } +} + +impl From for Version { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for Version { + fn from(value: usize) -> Self { + assert!(value <= u32::MAX as usize, "Version overflow: {value}"); + Self(value as u32) + } +} diff --git a/crates/vecdb/src/version/display.rs b/crates/vecdb/src/version/display.rs new file mode 100644 index 000000000..15f449409 --- /dev/null +++ b/crates/vecdb/src/version/display.rs @@ -0,0 +1,9 @@ +use std::fmt; + +use super::Version; + +impl fmt::Display for Version { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} diff --git a/crates/vecdb/src/version/mod.rs b/crates/vecdb/src/version/mod.rs new file mode 100644 index 000000000..fe97352e9 --- /dev/null +++ b/crates/vecdb/src/version/mod.rs @@ -0,0 +1,39 @@ +use std::{fs, io, path::Path}; + +mod add; +mod bytes; +mod conversions; +mod display; +mod sum; +mod try_from_path; + +use crate::Bytes; + +/// Version tracking for data schema and computed values. +/// +/// Used to detect when stored data needs to be recomputed due to changes +/// in computation logic or source data versions. Supports validation +/// against persisted versions to ensure compatibility. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[must_use = "Version values should be used for compatibility checks"] +pub struct Version(pub(super) u32); + +impl Version { + pub const ZERO: Self = Self(0); + pub const ONE: Self = Self(1); + pub const TWO: Self = Self(2); + + pub const fn new(v: u32) -> Self { + Self(v) + } + + pub fn write(&self, path: &Path) -> Result<(), io::Error> { + fs::write(path, self.to_bytes().as_ref()) + } + + pub fn swap_bytes(self) -> Self { + Self(self.0.swap_bytes()) + } +} diff --git a/crates/vecdb/src/version/sum.rs b/crates/vecdb/src/version/sum.rs new file mode 100644 index 000000000..b965ae3e7 --- /dev/null +++ b/crates/vecdb/src/version/sum.rs @@ -0,0 +1,9 @@ +use std::{iter::Sum, ops::Add}; + +use super::Version; + +impl Sum for Version { + fn sum>(iter: I) -> Self { + iter.fold(Self::ZERO, Add::add) + } +} diff --git a/crates/vecdb/src/version/try_from_path.rs b/crates/vecdb/src/version/try_from_path.rs new file mode 100644 index 000000000..4b38d3ca6 --- /dev/null +++ b/crates/vecdb/src/version/try_from_path.rs @@ -0,0 +1,14 @@ +use std::{fs, io::Read, path::Path}; + +use crate::{Bytes, Error}; + +use super::Version; + +impl TryFrom<&Path> for Version { + type Error = Error; + fn try_from(value: &Path) -> Result { + let mut buf = [0u8; size_of::()]; + fs::read(value)?.as_slice().read_exact(&mut buf)?; + Self::from_bytes(&buf) + } +} diff --git a/crates/vecdb/tests/basic.rs b/crates/vecdb/tests/basic.rs new file mode 100644 index 000000000..c256c4077 --- /dev/null +++ b/crates/vecdb/tests/basic.rs @@ -0,0 +1,261 @@ +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{Result, Stamp, StoredVec, Version}; + +/// Helper to create a temporary test database +pub fn setup_test_db() -> Result<(Database, TempDir)> { + let temp_dir = TempDir::new()?; + let db = Database::open(temp_dir.path())?; + Ok((db, temp_dir)) +} + +/// Generic test function for basic vec operations +fn run_vec_operations() -> Result<(), Box> +where + V: StoredVec, +{ + let version = Version::TWO; + let (database, _temp) = setup_test_db()?; + let options = (&database, "vec", version).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + vec.write()?; + + assert_eq!(vec.header().stamp(), Stamp::new(0)); + } + + { + let mut vec: V = V::forced_import_with(options)?; + + vec.mut_header().update_stamp(Stamp::new(100)); + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(3, 4), vec![3]); + assert_eq!(vec.collect_range(4, 5), vec![4]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(0, 1), vec![0]); + + vec.push(21); + vec.push(22); + + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.pushed_len(), 2); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + assert!(vec.collect_range(23, 24).is_empty()); + + vec.write()?; + } + + { + let mut vec: V = V::forced_import_with(options)?; + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.stored_len(), 23); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + + vec.truncate_if_needed(14)?; + + assert_eq!(vec.stored_len(), 14); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 14); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert!(vec.collect_range(20, 21).is_empty()); + + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![9, 10, 11, 12, 13] + ); + + vec.push(vec.len() as u32); + let last = vec.collect_range(vec.len() - 1, vec.len()); + assert_eq!(last[0], 14); + + vec.write()?; + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + } + + { + let mut vec: V = V::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert!(vec.collect_range(20, 21).is_empty()); + + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![10, 11, 12, 13, 14] + ); + + vec.reset()?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 0); + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.pushed_len(), 21); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 21); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + vec.write()?; + } + + { + let mut vec: V = V::forced_import_with(options)?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.len(), 21); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(10, 11), vec![10]); + + vec.write()?; + } + + { + let vec: V = V::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + } + + Ok(()) +} + +// ============================================================================ +// Test instantiation for each vec type +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} + +#[cfg(feature = "pco")] +mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} + +#[cfg(feature = "lz4")] +mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} + +#[cfg(feature = "zstd")] +mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} + +#[cfg(feature = "zerocopy")] +mod eager_zerocopy { + use super::*; + use vecdb::{EagerVec, ZeroCopyVec}; + type V = EagerVec>; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} + +#[cfg(feature = "pco")] +mod eager_pco { + use super::*; + use vecdb::{EagerVec, PcoVec}; + type V = EagerVec>; + + #[test] + fn test_vec_operations() -> Result<(), Box> { + run_vec_operations::() + } +} diff --git a/crates/vecdb/tests/basic2.rs b/crates/vecdb/tests/basic2.rs new file mode 100644 index 000000000..5e5aeeba7 --- /dev/null +++ b/crates/vecdb/tests/basic2.rs @@ -0,0 +1,592 @@ +//! Generic basic operations tests for all vec types. +//! +//! These tests run against any type implementing `StoredVec`, ensuring +//! consistent behavior across BytesVec, ZeroCopyVec, PcoVec, LZ4Vec, ZstdVec, and EagerVec. + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{Result, Stamp, StoredVec, Version}; + +// ============================================================================ +// Test Setup +// ============================================================================ + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +// ============================================================================ +// Generic Basic Operations Tests +// ============================================================================ + +fn run_push_write_read() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + // Push values + for i in 0..21_u32 { + vec.push(i); + } + + // Read via get before write + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + // Write to storage + vec.write()?; + + // Verify stamp + assert_eq!(vec.header().stamp(), Stamp::new(0)); + + Ok(()) +} + +fn run_stamp_management() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..21_u32 { + vec.push(i); + } + vec.write()?; + + // Reimport and update stamp + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + vec.mut_header().update_stamp(Stamp::new(100)); + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + // Verify data still readable + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + + Ok(()) +} + +fn run_length_tracking() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + // Push and write + for i in 0..21_u32 { + vec.push(i); + } + vec.write()?; + + // Reimport and verify + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 21); + + // Push more + vec.push(21); + vec.push(22); + + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.pushed_len(), 2); + assert_eq!(vec.len(), 23); + + // Read across stored/pushed boundary + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + assert!(vec.collect_range(23, 24).is_empty()); + + // Write and verify persistence + vec.write()?; + + let vec = V::forced_import(&db, "test", Version::ONE)?; + assert_eq!(vec.stored_len(), 23); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 23); + + Ok(()) +} + +fn run_truncate() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + // Push and write + for i in 0..23_u32 { + vec.push(i); + } + vec.write()?; + + // Reimport and truncate + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + vec.truncate_if_needed(14)?; + + assert_eq!(vec.stored_len(), 14); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 14); + + // Verify truncated data + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert_eq!(vec.collect_range(13, 14), vec![13]); + assert!(vec.collect_range(14, 15).is_empty()); + assert!(vec.collect_range(20, 21).is_empty()); + + Ok(()) +} + +fn run_collect_signed_range() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..15_u32 { + vec.push(i); + } + vec.write()?; + + // Test negative range (last 5 elements) + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![10, 11, 12, 13, 14] + ); + + // Test positive range + assert_eq!( + vec.collect_signed_range(Some(5), Some(10)), + vec![5, 6, 7, 8, 9] + ); + + Ok(()) +} + +fn run_iter_last() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..15_u32 { + vec.push(i); + } + vec.write()?; + + // Push one more without writing + vec.push(15); + + let last = vec.collect_range(vec.len() - 1, vec.len()); + assert_eq!(last[0], 15); + + Ok(()) +} + +fn run_reset() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..15_u32 { + vec.push(i); + } + vec.write()?; + + // Reset + vec.reset()?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 0); + + // Push new data after reset + for i in 0..21_u32 { + vec.push(i); + } + + assert_eq!(vec.pushed_len(), 21); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 21); + + // Verify new data + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + Ok(()) +} + +fn run_collect() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10_u32 { + vec.push(i); + } + vec.write()?; + + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + + Ok(()) +} + +fn run_persistence_across_reopen() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + + // Write data + { + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + for i in 0..100_u32 { + vec.push(i); + } + vec.write()?; + } + + // Reopen and verify + { + let vec = V::forced_import(&db, "test", Version::ONE)?; + assert_eq!(vec.len(), 100); + assert_eq!(vec.collect().len(), 100); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(50, 51), vec![50]); + assert_eq!(vec.collect_range(99, 100), vec![99]); + } + + Ok(()) +} + +// ============================================================================ +// Test instantiation for each vec type +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} + +#[cfg(feature = "pco")] +mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} + +#[cfg(feature = "lz4")] +mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} + +#[cfg(feature = "zstd")] +mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} + +#[cfg(feature = "zerocopy")] +mod eager_zerocopy { + use super::*; + use vecdb::{EagerVec, ZeroCopyVec}; + type V = EagerVec>; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} + +#[cfg(feature = "pco")] +mod eager_pco { + use super::*; + use vecdb::{EagerVec, PcoVec}; + type V = EagerVec>; + + #[test] + fn push_write_read() -> Result<()> { + run_push_write_read::() + } + #[test] + fn stamp_management() -> Result<()> { + run_stamp_management::() + } + #[test] + fn length_tracking() -> Result<()> { + run_length_tracking::() + } + #[test] + fn truncate() -> Result<()> { + run_truncate::() + } + #[test] + fn collect_signed_range() -> Result<()> { + run_collect_signed_range::() + } + #[test] + fn iter_last() -> Result<()> { + run_iter_last::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn collect() -> Result<()> { + run_collect::() + } + #[test] + fn persistence_across_reopen() -> Result<()> { + run_persistence_across_reopen::() + } +} diff --git a/crates/vecdb/tests/comprehensive.rs b/crates/vecdb/tests/comprehensive.rs new file mode 100644 index 000000000..6b0da221b --- /dev/null +++ b/crates/vecdb/tests/comprehensive.rs @@ -0,0 +1,656 @@ +//! Comprehensive tests for RawVec functionality (take, holes, update, collect_holed). +//! +//! These tests are specific to BytesVec and ZeroCopyVec which implement ReadWriteRawVec. + +use rawdb::Database; +use std::collections::BTreeSet; +use std::ops::DerefMut; +use tempfile::TempDir; +use vecdb::{ReadWriteRawVec, Result, Stamp, StoredVec, VecReader, Version}; + +// ============================================================================ +// Test Setup +// ============================================================================ + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +// ============================================================================ +// Trait for Raw Vec Operations +// ============================================================================ + +pub trait RawVecOps { + type Reader; + + fn take(&mut self, index: usize, reader: &Self::Reader) -> Option; + fn update(&mut self, index: usize, value: u32) -> Result<()>; + fn holes(&self) -> &BTreeSet; + fn collect_holed(&self) -> Vec>; + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option; + fn reader(&self) -> Self::Reader; +} + +impl RawVecOps for ReadWriteRawVec +where + S: vecdb::RawStrategy, +{ + type Reader = VecReader; + + fn take(&mut self, index: usize, reader: &Self::Reader) -> Option { + ReadWriteRawVec::take(self, index, reader) + } + + fn update(&mut self, index: usize, value: u32) -> Result<()> { + ReadWriteRawVec::update(self, index, value) + } + + fn holes(&self) -> &BTreeSet { + ReadWriteRawVec::holes(self) + } + + fn collect_holed(&self) -> Vec> { + ReadWriteRawVec::collect_holed(self) + } + + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option { + ReadWriteRawVec::get_with_reader(self, index, reader) + } + + fn reader(&self) -> Self::Reader { + ReadWriteRawVec::reader(self) + } +} + +// ============================================================================ +// Generic Comprehensive Tests +// ============================================================================ + +fn run_comprehensive_test() -> Result<()> +where + V: StoredVec + DerefMut, + V::Target: RawVecOps, +{ + let version = Version::TWO; + let (database, _temp) = setup_db()?; + let mut options = (&database, "vec", version).into(); + + { + let mut vec = V::forced_import_with(options)?; + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + vec.write()?; + + assert!(vec.header().stamp() == Stamp::new(0)); + } + + { + let mut vec = V::forced_import_with(options)?; + + vec.mut_header().update_stamp(Stamp::new(100)); + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(1, 2), vec![1]); + assert_eq!(vec.collect_range(2, 3), vec![2]); + assert_eq!(vec.collect_range(3, 4), vec![3]); + assert_eq!(vec.collect_range(4, 5), vec![4]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(0, 1), vec![0]); + + vec.push(21); + vec.push(22); + + assert_eq!(vec.stored_len(), 21); + assert_eq!(vec.pushed_len(), 2); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + assert!(vec.collect_range(23, 24).is_empty()); + + vec.write()?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!(vec.header().stamp(), Stamp::new(100)); + + assert_eq!(vec.stored_len(), 23); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 23); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert_eq!(vec.collect_range(21, 22), vec![21]); + assert_eq!(vec.collect_range(22, 23), vec![22]); + + vec.truncate_if_needed(14)?; + + assert_eq!(vec.stored_len(), 14); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.len(), 14); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(5, 6), vec![5]); + assert!(vec.collect_range(20, 21).is_empty()); + + assert_eq!( + vec.collect_signed_range(Some(-5), None), + vec![9, 10, 11, 12, 13] + ); + + vec.push(vec.len() as u32); + assert_eq!(vec.collect_range(vec.len() - 1, vec.len())[0], 14); + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + + vec.write()?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!(vec.collect_range(vec.len() - 1, vec.len())[0], 14); + + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ); + + vec.reset()?; + + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 0); + + (0..21_u32).for_each(|v| { + vec.push(v); + }); + + assert_eq!(vec.pushed_len(), 21); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 21); + + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(20, 21), vec![20]); + assert!(vec.collect_range(21, 22).is_empty()); + + let reader = vec.deref_mut().reader(); + assert_eq!(vec.deref_mut().take(10, &reader), Some(10)); + assert_eq!(vec.deref_mut().holes(), &BTreeSet::from([10])); + assert_eq!(vec.deref_mut().get_with_reader(10, &reader), None); + drop(reader); + + vec.write()?; + + assert!(vec.deref_mut().holes() == &BTreeSet::from([10])); + } + + { + let mut vec = V::forced_import_with(options)?; + + assert!(vec.deref_mut().holes() == &BTreeSet::from([10])); + + let reader = vec.deref_mut().reader(); + assert!(vec.deref_mut().get_with_reader(10, &reader).is_none()); + drop(reader); + + vec.deref_mut().update(10, 10)?; + vec.deref_mut().update(0, 10)?; + + let reader = vec.deref_mut().reader(); + assert_eq!(vec.deref_mut().holes(), &BTreeSet::new()); + assert_eq!(vec.deref_mut().get_with_reader(0, &reader), Some(10)); + assert_eq!(vec.deref_mut().get_with_reader(10, &reader), Some(10)); + drop(reader); + + vec.write()?; + } + + options = options.with_saved_stamped_changes(10); + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.truncate_if_needed(10)?; + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(5, &reader); + vec.deref_mut().update(3, 5)?; + vec.push(21); + drop(reader); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + Some(10), + Some(1), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(1))?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!(vec.collect(), vec![10, 1, 2, 5, 4, 6, 7, 8, 9, 21]); + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(0, &reader); + vec.deref_mut().update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(2))?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + vec.rollback()?; + + assert_eq!(vec.stamp(), Stamp::new(1)); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + Some(10), + Some(1), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21) + ] + ); + + vec.rollback()?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.stamped_write(Stamp::new(0))?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.truncate_if_needed(10)?; + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(5, &reader); + vec.deref_mut().update(3, 5)?; + vec.push(21); + drop(reader); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + Some(10), + Some(1), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(1))?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!(vec.collect(), vec![10, 1, 2, 5, 4, 6, 7, 8, 9, 21]); + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(0, &reader); + vec.deref_mut().update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + vec.stamped_write_with_changes(Stamp::new(2))?; + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + let _ = vec.rollback_before(Stamp::new(1))?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.stamped_write(Stamp::new(0))?; + + vec.truncate_if_needed(10)?; + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(5, &reader); + vec.deref_mut().update(3, 5)?; + vec.push(21); + drop(reader); + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(0, &reader); + vec.deref_mut().update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + } + + { + let mut vec = V::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.truncate_if_needed(10)?; + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(5, &reader); + vec.deref_mut().update(3, 5)?; + vec.push(21); + drop(reader); + + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.stamp(), Stamp::new(1)); + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(0, &reader); + vec.deref_mut().update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + vec.stamped_write_with_changes(Stamp::new(2))?; + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + let _ = vec.rollback_before(Stamp::new(1))?; + + assert_eq!(vec.stamp(), Stamp::new(0)); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.truncate_if_needed(10)?; + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(5, &reader); + vec.deref_mut().update(3, 5)?; + vec.push(21); + drop(reader); + + let reader = vec.deref_mut().reader(); + let _ = vec.deref_mut().take(0, &reader); + vec.deref_mut().update(1, 5)?; + vec.push(5); + vec.push(6); + vec.push(7); + drop(reader); + + assert_eq!( + vec.deref_mut().collect_holed(), + vec![ + None, + Some(5), + Some(2), + Some(5), + Some(4), + None, + Some(6), + Some(7), + Some(8), + Some(9), + Some(21), + Some(5), + Some(6), + Some(7) + ] + ); + + assert_eq!(vec.stamp(), Stamp::new(0)); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.stamp(), Stamp::new(2)); + + let _ = vec.rollback_before(Stamp::new(1))?; + + assert_eq!(vec.stamp(), Stamp::new(0)); + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + + vec.stamped_write_with_changes(Stamp::new(0))?; + + let vec = V::forced_import_with(options)?; + + assert_eq!( + vec.collect(), + vec![ + 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + ] + ); + } + + Ok(()) +} + +// ============================================================================ +// Test instantiation for BytesVec and ZeroCopyVec +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn test_raw_vec_comprehensive() -> Result<()> { + run_comprehensive_test::() + } +} + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn test_raw_vec_comprehensive() -> Result<()> { + run_comprehensive_test::() + } +} diff --git a/crates/vecdb/tests/compute_functions.rs b/crates/vecdb/tests/compute_functions.rs new file mode 100644 index 000000000..9021d4ff4 --- /dev/null +++ b/crates/vecdb/tests/compute_functions.rs @@ -0,0 +1,1477 @@ +//! Generic compute function tests for all vec types. +//! +//! These tests run against any type implementing `StoredVec`, ensuring +//! consistent compute behavior across all vec types. + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, EagerVec, Exit, ImportableVec, ReadableVec, Result, StoredVec, Version, + WritableVec, +}; + +// ============================================================================ +// Test Setup +// ============================================================================ + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +fn assert_f32_eq(actual: f32, expected: f32, tolerance: f32, message: &str) { + assert!( + (actual - expected).abs() < tolerance, + "{}: expected {}, got {} (diff: {})", + message, + expected, + actual, + (actual - expected).abs() + ); +} + +// ============================================================================ +// Generic Test Functions +// ============================================================================ + +fn run_compute_sum_of_others() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + let mut vec3: EagerVec = EagerVec::forced_import(&db, "vec3", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (i * 10) as u64)?; + vec2.checked_push(i, (i * 5) as u64)?; + vec3.checked_push(i, i as u64)?; + } + vec1.flush()?; + vec2.flush()?; + vec3.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_sum_of_others(0, &[&vec1, &vec2, &vec3], &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = ((i * 10) + (i * 5) + i) as u64; + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, expected, + "Sum mismatch at index {}: expected {}, got {}", + i, expected, actual + ); + } + + Ok(()) +} + +fn run_compute_min_of_others() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + let mut vec3: EagerVec = EagerVec::forced_import(&db, "vec3", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (50 + i) as u64)?; + vec2.checked_push(i, (10 + i) as u64)?; + vec3.checked_push(i, (100 + i) as u64)?; + } + vec1.flush()?; + vec2.flush()?; + vec3.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_min_of_others(0, &[&vec1, &vec2, &vec3], &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = (10 + i) as u64; + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, expected, + "Min mismatch at index {}: expected {}, got {}", + i, expected, actual + ); + } + + Ok(()) +} + +fn run_compute_max_of_others() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + let mut vec3: EagerVec = EagerVec::forced_import(&db, "vec3", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (50 + i) as u64)?; + vec2.checked_push(i, (10 + i) as u64)?; + vec3.checked_push(i, (100 + i) as u64)?; + } + vec1.flush()?; + vec2.flush()?; + vec3.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_max_of_others(0, &[&vec1, &vec2, &vec3], &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = (100 + i) as u64; + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, expected, + "Max mismatch at index {}: expected {}, got {}", + i, expected, actual + ); + } + + Ok(()) +} + +fn run_compute_previous_value() -> Result<()> +where + VS: StoredVec, + VR: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source = EagerVec::::forced_import(&db, "source", Version::ONE)?; + + for i in 0..5 { + source.checked_push(i, ((i + 1) * 10) as u16)?; + } + source.flush()?; + + let mut result = EagerVec::::forced_import(&db, "result", Version::ONE)?; + result.compute_previous_value(0, &source, 1, &exit)?; + result.flush()?; + + let actual_0 = result.collect_first().unwrap(); + assert!( + actual_0.is_nan(), + "First element should be NaN when no previous value exists" + ); + + let expected = [10.0, 20.0, 30.0, 40.0]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i + 1).unwrap(); + assert_eq!( + actual, + v, + "Previous value mismatch at index {}: expected {}, got {}", + i + 1, + v, + actual + ); + } + + Ok(()) +} + +fn run_compute_change() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + let values = [10, 20, 25, 30, 50]; + for (i, &v) in values.iter().enumerate() { + source.checked_push(i, v)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_change(0, &source, 1, &exit)?; + result.flush()?; + + let expected = [0, 10, 5, 5, 20]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, v, + "Change mismatch at index {}: expected {}, got {}", + i, v, actual + ); + } + + Ok(()) +} + +fn run_compute_percentage_change() -> Result<()> +where + VS: StoredVec, + VR: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + let values = [100, 110, 121, 133]; + for (i, &v) in values.iter().enumerate() { + source.checked_push(i, v)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_percentage_change(0, &source, 1, &exit)?; + result.flush()?; + + let actual_0 = result.collect_first().unwrap(); + let actual_1 = result.collect_one(1).unwrap(); + let actual_2 = result.collect_one(2).unwrap(); + let actual_3 = result.collect_one(3).unwrap(); + + assert!( + actual_0.is_nan(), + "First element should be NaN when no previous value exists" + ); + assert!((actual_1 - 10.0).abs() < 0.01); + assert!((actual_2 - 10.0).abs() < 0.01); + assert!((actual_3 - 9.917).abs() < 0.01); + + Ok(()) +} + +fn run_compute_sliding_window_max() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + let values = [3, 1, 4, 1, 5, 9, 2, 6]; + for (i, &v) in values.iter().enumerate() { + source.checked_push(i, v)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_max(0, &source, 3, &exit)?; + result.flush()?; + + let expected = [3, 3, 4, 4, 5, 9, 9, 9]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, v, + "Sliding window max mismatch at index {}: expected {}, got {}", + i, v, actual + ); + } + + Ok(()) +} + +fn run_compute_sliding_window_min() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + let values = [3, 1, 4, 1, 5, 9, 2, 6]; + for (i, &v) in values.iter().enumerate() { + source.checked_push(i, v)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_min(0, &source, 3, &exit)?; + result.flush()?; + + let expected = [3, 1, 1, 1, 1, 1, 2, 2]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, v, + "Sliding window min mismatch at index {}: expected {}, got {}", + i, v, actual + ); + } + + Ok(()) +} + +fn run_compute_all_time_high() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + let values = [10, 15, 12, 20, 18, 25, 22]; + for (i, &v) in values.iter().enumerate() { + source.checked_push(i, v)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_all_time_high(0, &source, &exit)?; + result.flush()?; + + let expected = [10, 15, 15, 20, 20, 25, 25]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, v, + "All-time high mismatch at index {}: expected {}, got {}", + i, v, actual + ); + } + + Ok(()) +} + +fn run_compute_all_time_low() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + let values = [10, 5, 12, 3, 18, 2, 22]; + for (i, &v) in values.iter().enumerate() { + source.checked_push(i, v)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_all_time_low_(0, &source, &exit, false)?; + result.flush()?; + + let expected = [10, 5, 5, 3, 3, 2, 2]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i).unwrap(); + assert_eq!( + actual, v, + "All-time low mismatch at index {}: expected {}, got {}", + i, v, actual + ); + } + + Ok(()) +} + +fn run_compute_cagr() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut percentage_returns: EagerVec = + EagerVec::forced_import(&db, "returns", Version::ONE)?; + + for i in 0..5 { + percentage_returns.checked_push(i, 100.0)?; + } + percentage_returns.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_cagr(0, &percentage_returns, 730, &exit)?; + result.flush()?; + + for i in 0..5 { + let actual = result.collect_one(i).unwrap(); + let expected = 41.42; + assert!( + (actual - expected).abs() < 0.01, + "CAGR mismatch at index {}: expected {}, got {}", + i, + expected, + actual + ); + } + + Ok(()) +} + +fn run_compute_zscore() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + let mut sma: EagerVec = EagerVec::forced_import(&db, "sma", Version::ONE)?; + let mut sd: EagerVec = EagerVec::forced_import(&db, "sd", Version::ONE)?; + + for i in 0..4 { + source.checked_push(i, 10.0 + i as f32 * 2.0)?; + sma.checked_push(i, 10.0)?; + sd.checked_push(i, 2.0)?; + } + source.flush()?; + sma.flush()?; + sd.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_zscore(0, &source, &sma, &sd, &exit)?; + result.flush()?; + + let expected = [0.0, 1.0, 2.0, 3.0]; + for (i, v) in expected.into_iter().enumerate() { + let actual = result.collect_one(i).unwrap(); + assert!( + (actual - v).abs() < 0.01, + "Z-score mismatch at index {}: expected {}, got {}", + i, + v, + actual + ); + } + + Ok(()) +} + +fn run_compute_functions_with_resume() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + + for i in 0..5 { + source.checked_push(i, (i * 10) as u32)?; + } + source.flush()?; + + result.compute_all_time_high(0, &source, &exit)?; + result.flush()?; + + for i in 0..5 { + let actual = result.collect_one(i).unwrap(); + let expected = (i * 10) as u32; + assert_eq!(actual, expected); + } + + for i in 5..10 { + source.checked_push(i, (i * 10) as u32)?; + } + source.flush()?; + + result.compute_all_time_high(0, &source, &exit)?; + result.flush()?; + + for i in 0..10 { + let actual = result.collect_one(i).unwrap(); + let expected = (i * 10) as u32; + assert_eq!(actual, expected); + } + + Ok(()) +} + +fn run_compute_add() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (i * 10) as u64)?; + vec2.checked_push(i, (i * 5) as u64)?; + } + vec1.flush()?; + vec2.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_add(0, &vec1, &vec2, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = (i * 10 + i * 5) as u64; + let actual = result.collect_one(i).unwrap(); + assert_eq!(actual, expected); + } + + Ok(()) +} + +fn run_compute_subtract() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (100 + i * 10) as u64)?; + vec2.checked_push(i, (i * 5) as u64)?; + } + vec1.flush()?; + vec2.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_subtract(0, &vec1, &vec2, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = (100 + i * 10 - i * 5) as u64; + let actual = result.collect_one(i).unwrap(); + assert_eq!(actual, expected); + } + + Ok(()) +} + +fn run_compute_multiply() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (i + 1) as u32)?; + vec2.checked_push(i, (i + 2) as u32)?; + } + vec1.flush()?; + vec2.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_multiply(0, &vec1, &vec2, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = ((i + 1) * (i + 2)) as u32; + let actual = result.collect_one(i).unwrap(); + assert_eq!(actual, expected); + } + + Ok(()) +} + +fn run_compute_divide() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, 100.0 + i as f32 * 10.0)?; + vec2.checked_push(i, i as f32 + 1.0)?; + } + vec1.flush()?; + vec2.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_divide(0, &vec1, &vec2, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = (100.0 + i as f32 * 10.0) / (i as f32 + 1.0); + let actual = result.collect_one(i).unwrap(); + assert_f32_eq(actual, expected, 0.001, &format!("Divide at index {}", i)); + } + + Ok(()) +} + +fn run_compute_max() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + for i in 0..10 { + let value = if i < 5 { i * 10 } else { (9 - i) * 10 }; + source.checked_push(i, value as u64)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_max(0, &source, usize::MAX, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = if i < 5 { (i * 10) as u64 } else { 40u64 }; + let actual = result.collect_one(i).unwrap(); + assert_eq!(actual, expected, "Max at index {}", i); + } + + Ok(()) +} + +fn run_compute_min() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + for i in 0..10 { + let value = if i < 5 { + 100 - i * 10 + } else { + 50 + (i - 5) * 10 + }; + source.checked_push(i, value as u64)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_min(0, &source, usize::MAX, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = if i < 5 { (100 - i * 10) as u64 } else { 50u64 }; + let actual = result.collect_one(i).unwrap(); + assert_eq!(actual, expected, "Min at index {}", i); + } + + Ok(()) +} + +fn run_compute_sum() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + for i in 0..10 { + source.checked_push(i, (i + 1) as u64)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_sum(0, &source, usize::MAX, &exit)?; + result.flush()?; + + let mut expected_sum = 0u64; + for i in 0..10 { + expected_sum += (i + 1) as u64; + let actual = result.collect_one(i).unwrap(); + assert_eq!(actual, expected_sum, "Cumulative sum at index {}", i); + } + + Ok(()) +} + +fn run_compute_sma() -> Result<()> +where + VS: StoredVec, + VR: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + for i in 0..10 { + source.checked_push(i, (i * 10) as u16)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_sma(0, &source, 3, &exit)?; + result.flush()?; + + for i in 0..10_u64 { + let actual = result.collect_one(i as usize).unwrap(); + if i < 2 { + let sum: u64 = (0..=i).map(|j| j * 10).sum(); + let expected = sum as f32 / (i + 1) as f32; + assert_f32_eq(actual, expected, 0.001, &format!("SMA at index {}", i)); + } else { + let sum: u64 = (i - 2..=i).map(|j| j * 10).sum(); + let expected = sum as f32 / 3.0; + assert_f32_eq(actual, expected, 0.001, &format!("SMA at index {}", i)); + } + } + + Ok(()) +} + +fn run_compute_ema() -> Result<()> +where + VS: StoredVec, + VR: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut source: EagerVec = EagerVec::forced_import(&db, "source", Version::ONE)?; + + for i in 0..10 { + source.checked_push(i, 100)?; + } + source.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_ema(0, &source, 3, &exit)?; + result.flush()?; + + for i in 0..10 { + let actual = result.collect_one(i).unwrap(); + assert_f32_eq(actual, 100.0, 0.1, &format!("EMA at index {}", i)); + } + + Ok(()) +} + +fn run_compute_percentage() -> Result<()> +where + VS: StoredVec, + VR: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut numerator: EagerVec = EagerVec::forced_import(&db, "numerator", Version::ONE)?; + let mut denominator: EagerVec = EagerVec::forced_import(&db, "denominator", Version::ONE)?; + + for i in 0..10 { + numerator.checked_push(i, (i + 1) as u16)?; + denominator.checked_push(i, 10)?; + } + numerator.flush()?; + denominator.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_percentage(0, &numerator, &denominator, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = ((i + 1) as f32 / 10.0) * 100.0; + let actual = result.collect_one(i).unwrap(); + assert_f32_eq( + actual, + expected, + 0.001, + &format!("Percentage at index {}", i), + ); + } + + Ok(()) +} + +fn run_compute_percentage_difference() -> Result<()> +where + VS: StoredVec, + VR: StoredVec, +{ + let (db, _temp) = setup_db()?; + let exit = Exit::new(); + + let mut vec1: EagerVec = EagerVec::forced_import(&db, "vec1", Version::ONE)?; + let mut vec2: EagerVec = EagerVec::forced_import(&db, "vec2", Version::ONE)?; + + for i in 0..10 { + vec1.checked_push(i, (100 + i * 10) as u16)?; + vec2.checked_push(i, 100)?; + } + vec1.flush()?; + vec2.flush()?; + + let mut result: EagerVec = EagerVec::forced_import(&db, "result", Version::ONE)?; + result.compute_percentage_difference(0, &vec1, &vec2, &exit)?; + result.flush()?; + + for i in 0..10 { + let expected = (((100 + i * 10) as f32 - 100.0) / 100.0) * 100.0; + let actual = result.collect_one(i).unwrap(); + assert_f32_eq( + actual, + expected, + 0.001, + &format!("Percentage difference at index {}", i), + ); + } + + Ok(()) +} + +// ============================================================================ +// Test instantiation for BytesVec (no feature flag needed) +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + + #[test] + fn compute_sum_of_others() -> Result<()> { + run_compute_sum_of_others::>() + } + + #[test] + fn compute_min_of_others() -> Result<()> { + run_compute_min_of_others::>() + } + + #[test] + fn compute_max_of_others() -> Result<()> { + run_compute_max_of_others::>() + } + + #[test] + fn compute_previous_value() -> Result<()> { + run_compute_previous_value::, BytesVec>() + } + + #[test] + fn compute_change() -> Result<()> { + run_compute_change::>() + } + + #[test] + fn compute_percentage_change() -> Result<()> { + run_compute_percentage_change::, BytesVec>() + } + + #[test] + fn compute_sliding_window_max() -> Result<()> { + run_compute_sliding_window_max::>() + } + + #[test] + fn compute_sliding_window_min() -> Result<()> { + run_compute_sliding_window_min::>() + } + + #[test] + fn compute_all_time_high() -> Result<()> { + run_compute_all_time_high::>() + } + + #[test] + fn compute_all_time_low() -> Result<()> { + run_compute_all_time_low::>() + } + + #[test] + fn compute_cagr() -> Result<()> { + run_compute_cagr::>() + } + + #[test] + fn compute_zscore() -> Result<()> { + run_compute_zscore::>() + } + + #[test] + fn compute_functions_with_resume() -> Result<()> { + run_compute_functions_with_resume::>() + } + + #[test] + fn compute_add() -> Result<()> { + run_compute_add::>() + } + + #[test] + fn compute_subtract() -> Result<()> { + run_compute_subtract::>() + } + + #[test] + fn compute_multiply() -> Result<()> { + run_compute_multiply::>() + } + + #[test] + fn compute_divide() -> Result<()> { + run_compute_divide::>() + } + + #[test] + fn compute_max() -> Result<()> { + run_compute_max::>() + } + + #[test] + fn compute_min() -> Result<()> { + run_compute_min::>() + } + + #[test] + fn compute_sum() -> Result<()> { + run_compute_sum::>() + } + + #[test] + fn compute_sma() -> Result<()> { + run_compute_sma::, BytesVec>() + } + + #[test] + fn compute_ema() -> Result<()> { + run_compute_ema::, BytesVec>() + } + + #[test] + fn compute_percentage() -> Result<()> { + run_compute_percentage::, BytesVec>() + } + + #[test] + fn compute_percentage_difference() -> Result<()> { + run_compute_percentage_difference::, BytesVec>() + } +} + +// ============================================================================ +// Test instantiation for feature-gated vec types +// ============================================================================ + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + + #[test] + fn compute_sum_of_others() -> Result<()> { + run_compute_sum_of_others::>() + } + + #[test] + fn compute_min_of_others() -> Result<()> { + run_compute_min_of_others::>() + } + + #[test] + fn compute_max_of_others() -> Result<()> { + run_compute_max_of_others::>() + } + + #[test] + fn compute_previous_value() -> Result<()> { + run_compute_previous_value::, ZeroCopyVec>() + } + + #[test] + fn compute_change() -> Result<()> { + run_compute_change::>() + } + + #[test] + fn compute_percentage_change() -> Result<()> { + run_compute_percentage_change::, ZeroCopyVec>() + } + + #[test] + fn compute_sliding_window_max() -> Result<()> { + run_compute_sliding_window_max::>() + } + + #[test] + fn compute_sliding_window_min() -> Result<()> { + run_compute_sliding_window_min::>() + } + + #[test] + fn compute_all_time_high() -> Result<()> { + run_compute_all_time_high::>() + } + + #[test] + fn compute_all_time_low() -> Result<()> { + run_compute_all_time_low::>() + } + + #[test] + fn compute_cagr() -> Result<()> { + run_compute_cagr::>() + } + + #[test] + fn compute_zscore() -> Result<()> { + run_compute_zscore::>() + } + + #[test] + fn compute_functions_with_resume() -> Result<()> { + run_compute_functions_with_resume::>() + } + + #[test] + fn compute_add() -> Result<()> { + run_compute_add::>() + } + + #[test] + fn compute_subtract() -> Result<()> { + run_compute_subtract::>() + } + + #[test] + fn compute_multiply() -> Result<()> { + run_compute_multiply::>() + } + + #[test] + fn compute_divide() -> Result<()> { + run_compute_divide::>() + } + + #[test] + fn compute_max() -> Result<()> { + run_compute_max::>() + } + + #[test] + fn compute_min() -> Result<()> { + run_compute_min::>() + } + + #[test] + fn compute_sum() -> Result<()> { + run_compute_sum::>() + } + + #[test] + fn compute_sma() -> Result<()> { + run_compute_sma::, ZeroCopyVec>() + } + + #[test] + fn compute_ema() -> Result<()> { + run_compute_ema::, ZeroCopyVec>() + } + + #[test] + fn compute_percentage() -> Result<()> { + run_compute_percentage::, ZeroCopyVec>() + } + + #[test] + fn compute_percentage_difference() -> Result<()> { + run_compute_percentage_difference::, ZeroCopyVec>() + } +} + +#[cfg(feature = "pco")] +mod pco { + use super::*; + use vecdb::PcoVec; + + #[test] + fn compute_sum_of_others() -> Result<()> { + run_compute_sum_of_others::>() + } + + #[test] + fn compute_min_of_others() -> Result<()> { + run_compute_min_of_others::>() + } + + #[test] + fn compute_max_of_others() -> Result<()> { + run_compute_max_of_others::>() + } + + #[test] + fn compute_previous_value() -> Result<()> { + run_compute_previous_value::, PcoVec>() + } + + #[test] + fn compute_change() -> Result<()> { + run_compute_change::>() + } + + #[test] + fn compute_percentage_change() -> Result<()> { + run_compute_percentage_change::, PcoVec>() + } + + #[test] + fn compute_sliding_window_max() -> Result<()> { + run_compute_sliding_window_max::>() + } + + #[test] + fn compute_sliding_window_min() -> Result<()> { + run_compute_sliding_window_min::>() + } + + #[test] + fn compute_all_time_high() -> Result<()> { + run_compute_all_time_high::>() + } + + #[test] + fn compute_all_time_low() -> Result<()> { + run_compute_all_time_low::>() + } + + #[test] + fn compute_cagr() -> Result<()> { + run_compute_cagr::>() + } + + #[test] + fn compute_zscore() -> Result<()> { + run_compute_zscore::>() + } + + #[test] + fn compute_functions_with_resume() -> Result<()> { + run_compute_functions_with_resume::>() + } + + #[test] + fn compute_add() -> Result<()> { + run_compute_add::>() + } + + #[test] + fn compute_subtract() -> Result<()> { + run_compute_subtract::>() + } + + #[test] + fn compute_multiply() -> Result<()> { + run_compute_multiply::>() + } + + #[test] + fn compute_divide() -> Result<()> { + run_compute_divide::>() + } + + #[test] + fn compute_max() -> Result<()> { + run_compute_max::>() + } + + #[test] + fn compute_min() -> Result<()> { + run_compute_min::>() + } + + #[test] + fn compute_sum() -> Result<()> { + run_compute_sum::>() + } + + #[test] + fn compute_sma() -> Result<()> { + run_compute_sma::, PcoVec>() + } + + #[test] + fn compute_ema() -> Result<()> { + run_compute_ema::, PcoVec>() + } + + #[test] + fn compute_percentage() -> Result<()> { + run_compute_percentage::, PcoVec>() + } + + #[test] + fn compute_percentage_difference() -> Result<()> { + run_compute_percentage_difference::, PcoVec>() + } +} + +#[cfg(feature = "lz4")] +mod lz4 { + use super::*; + use vecdb::LZ4Vec; + + #[test] + fn compute_sum_of_others() -> Result<()> { + run_compute_sum_of_others::>() + } + + #[test] + fn compute_min_of_others() -> Result<()> { + run_compute_min_of_others::>() + } + + #[test] + fn compute_max_of_others() -> Result<()> { + run_compute_max_of_others::>() + } + + #[test] + fn compute_previous_value() -> Result<()> { + run_compute_previous_value::, LZ4Vec>() + } + + #[test] + fn compute_change() -> Result<()> { + run_compute_change::>() + } + + #[test] + fn compute_percentage_change() -> Result<()> { + run_compute_percentage_change::, LZ4Vec>() + } + + #[test] + fn compute_sliding_window_max() -> Result<()> { + run_compute_sliding_window_max::>() + } + + #[test] + fn compute_sliding_window_min() -> Result<()> { + run_compute_sliding_window_min::>() + } + + #[test] + fn compute_all_time_high() -> Result<()> { + run_compute_all_time_high::>() + } + + #[test] + fn compute_all_time_low() -> Result<()> { + run_compute_all_time_low::>() + } + + #[test] + fn compute_cagr() -> Result<()> { + run_compute_cagr::>() + } + + #[test] + fn compute_zscore() -> Result<()> { + run_compute_zscore::>() + } + + #[test] + fn compute_functions_with_resume() -> Result<()> { + run_compute_functions_with_resume::>() + } + + #[test] + fn compute_add() -> Result<()> { + run_compute_add::>() + } + + #[test] + fn compute_subtract() -> Result<()> { + run_compute_subtract::>() + } + + #[test] + fn compute_multiply() -> Result<()> { + run_compute_multiply::>() + } + + #[test] + fn compute_divide() -> Result<()> { + run_compute_divide::>() + } + + #[test] + fn compute_max() -> Result<()> { + run_compute_max::>() + } + + #[test] + fn compute_min() -> Result<()> { + run_compute_min::>() + } + + #[test] + fn compute_sum() -> Result<()> { + run_compute_sum::>() + } + + #[test] + fn compute_sma() -> Result<()> { + run_compute_sma::, LZ4Vec>() + } + + #[test] + fn compute_ema() -> Result<()> { + run_compute_ema::, LZ4Vec>() + } + + #[test] + fn compute_percentage() -> Result<()> { + run_compute_percentage::, LZ4Vec>() + } + + #[test] + fn compute_percentage_difference() -> Result<()> { + run_compute_percentage_difference::, LZ4Vec>() + } +} + +#[cfg(feature = "zstd")] +mod zstd { + use super::*; + use vecdb::ZstdVec; + + #[test] + fn compute_sum_of_others() -> Result<()> { + run_compute_sum_of_others::>() + } + + #[test] + fn compute_min_of_others() -> Result<()> { + run_compute_min_of_others::>() + } + + #[test] + fn compute_max_of_others() -> Result<()> { + run_compute_max_of_others::>() + } + + #[test] + fn compute_previous_value() -> Result<()> { + run_compute_previous_value::, ZstdVec>() + } + + #[test] + fn compute_change() -> Result<()> { + run_compute_change::>() + } + + #[test] + fn compute_percentage_change() -> Result<()> { + run_compute_percentage_change::, ZstdVec>() + } + + #[test] + fn compute_sliding_window_max() -> Result<()> { + run_compute_sliding_window_max::>() + } + + #[test] + fn compute_sliding_window_min() -> Result<()> { + run_compute_sliding_window_min::>() + } + + #[test] + fn compute_all_time_high() -> Result<()> { + run_compute_all_time_high::>() + } + + #[test] + fn compute_all_time_low() -> Result<()> { + run_compute_all_time_low::>() + } + + #[test] + fn compute_cagr() -> Result<()> { + run_compute_cagr::>() + } + + #[test] + fn compute_zscore() -> Result<()> { + run_compute_zscore::>() + } + + #[test] + fn compute_functions_with_resume() -> Result<()> { + run_compute_functions_with_resume::>() + } + + #[test] + fn compute_add() -> Result<()> { + run_compute_add::>() + } + + #[test] + fn compute_subtract() -> Result<()> { + run_compute_subtract::>() + } + + #[test] + fn compute_multiply() -> Result<()> { + run_compute_multiply::>() + } + + #[test] + fn compute_divide() -> Result<()> { + run_compute_divide::>() + } + + #[test] + fn compute_max() -> Result<()> { + run_compute_max::>() + } + + #[test] + fn compute_min() -> Result<()> { + run_compute_min::>() + } + + #[test] + fn compute_sum() -> Result<()> { + run_compute_sum::>() + } + + #[test] + fn compute_sma() -> Result<()> { + run_compute_sma::, ZstdVec>() + } + + #[test] + fn compute_ema() -> Result<()> { + run_compute_ema::, ZstdVec>() + } + + #[test] + fn compute_percentage() -> Result<()> { + run_compute_percentage::, ZstdVec>() + } + + #[test] + fn compute_percentage_difference() -> Result<()> { + run_compute_percentage_difference::, ZstdVec>() + } +} diff --git a/crates/vecdb/tests/concurrent_rw.rs b/crates/vecdb/tests/concurrent_rw.rs new file mode 100644 index 000000000..fd0061387 --- /dev/null +++ b/crates/vecdb/tests/concurrent_rw.rs @@ -0,0 +1,1016 @@ +//! Tests for concurrent read/write behavior with batched syncs. +//! +//! These tests verify that: +//! 1. A reader clone can see data written by writer after write() (even without sync) +//! 2. Batched writes followed by a single sync are safe for concurrent readers +//! 3. The SharedLen atomic is properly visible across clones +//! 4. Data written to mmap is visible to readers BEFORE stored_len is updated +//! (critical for memory ordering correctness) +//! +//! IMPORTANT: These tests verify that the write ordering is correct: +//! 1. Data must be written to mmap +//! 2. Memory barrier (implicit in SeqCst atomic) +//! 3. stored_len is updated +//! +//! If a reader sees a new stored_len, the corresponding data MUST be readable. + +use std::{ + sync::{ + Arc, Barrier, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + thread, + time::Duration, +}; + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, AnyVec, BytesVec, ImportableVec, ReadableVec, Result, StoredVec, Version, + WritableVec, +}; + +#[cfg(feature = "pco")] +use vecdb::PcoVec; + +fn setup_test_db() -> Result<(Database, TempDir)> { + let temp_dir = TempDir::new()?; + let db = Database::open(temp_dir.path())?; + Ok((db, temp_dir)) +} + +/// Test that a cloned reader can see data after writer calls write() but before flush() +#[test] +fn test_reader_sees_written_data_without_flush() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + // Create writer vec and write initial data + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + for i in 0..100u64 { + writer.push(i); + } + writer.write()?; + // Note: NOT calling flush() here - data is in mmap but not synced to disk + + // Create VecReader (simulates what Query does) + let r = writer.reader(); + + // Reader should see the stored data via shared mmap + assert_eq!(r.len(), 100); + assert_eq!(r.get(0), 0); + assert_eq!(r.get(50), 50); + assert_eq!(r.get(99), 99); + + Ok(()) +} + +/// Test that reader sees new data written after clone, once write() is called +#[test] +fn test_reader_sees_new_data_after_write() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + // Create and initialize writer + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + for i in 0..50u64 { + writer.push(i); + } + writer.write()?; + + // Create read-only clone sharing SharedLen + let reader = writer.read_only_clone(); + assert_eq!(reader.len(), 50); + + // Writer adds more data + for i in 50..100u64 { + writer.push(i); + } + + // Reader still sees old stored_len (pushed is not shared) + assert_eq!(reader.len(), 50); + + // Writer calls write() - this updates stored_len (shared) and writes to mmap + writer.write()?; + + // Now reader should see the new stored_len + assert_eq!(reader.len(), 100); + + // And read-only clone can create a VecReader for O(1) point reads + let r = reader.reader(); + assert_eq!(r.get(99), 99); + assert_eq!(r.get(75), 75); + + Ok(()) +} + +/// Test concurrent read while write is happening +#[test] +fn test_concurrent_read_during_write() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + // Write initial batch + for i in 0..1000u64 { + writer.push(i); + } + writer.write()?; + + let reader = writer.read_only_clone(); + let barrier = Arc::new(Barrier::new(2)); + + let reader_barrier = barrier.clone(); + let reader_handle = thread::spawn(move || -> Result<()> { + // Wait for writer to start + reader_barrier.wait(); + + // Continuously read while writer is working + for _ in 0..100 { + let len = reader.len(); + if len > 0 { + let r = reader.reader(); + // Read some values - should never panic or return garbage + for i in 0..len.min(100) { + let val = r.try_get(i); + assert!(val.is_some(), "Expected value at index {}", i); + assert_eq!(val.unwrap(), i as u64); + } + } + thread::sleep(Duration::from_micros(100)); + } + Ok(()) + }); + + // Signal reader to start, then write more data + barrier.wait(); + + for batch in 0..10 { + for i in 0..100u64 { + writer.push(1000 + batch * 100 + i); + } + writer.write()?; + thread::sleep(Duration::from_micros(50)); + } + + reader_handle.join().unwrap()?; + + // Final state check + assert_eq!(writer.len(), 2000); + + Ok(()) +} + +/// Test that multiple vecs can be written without flush, then flushed together +#[test] +fn test_batched_writes_single_flush() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut vec1: BytesVec = BytesVec::forced_import(&db, "vec1", version)?; + let mut vec2: BytesVec = BytesVec::forced_import(&db, "vec2", version)?; + let mut vec3: BytesVec = BytesVec::forced_import(&db, "vec3", version)?; + + // Write to all vecs without flushing + for i in 0..100u64 { + vec1.push(i); + vec2.push(i * 2); + vec3.push(i * 3); + } + + // Write all (to mmap) without flush + vec1.write()?; + vec2.write()?; + vec3.write()?; + + // Create VecReaders + let r1 = vec1.reader(); + let r2 = vec2.reader(); + let r3 = vec3.reader(); + + // All readers should see the data + assert_eq!(r1.len(), 100); + assert_eq!(r2.len(), 100); + assert_eq!(r3.len(), 100); + + assert_eq!(r1.get(50), 50); + assert_eq!(r2.get(50), 100); + assert_eq!(r3.get(50), 150); + + // Flush while readers are still alive - no deadlock since + // dirty_range is in a separate Mutex from region metadata + db.flush()?; + + // Data should still be readable after flush + drop(r1); + drop(r2); + drop(r3); + + let r1 = vec1.reader(); + assert_eq!(r1.get(99), 99); + + Ok(()) +} + +/// Test with PcoVec (compressed) to ensure it also works +#[test] +#[cfg(feature = "pco")] +fn test_pco_concurrent_read_write() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: PcoVec = PcoVec::forced_import(&db, "pco_vec", version)?; + + for i in 0..500u64 { + writer.push(i); + } + writer.write()?; + + let reader = writer.read_only_clone(); + + // Add more data + for i in 500..1000u64 { + writer.push(i); + } + writer.write()?; + + // Reader should see all data + assert_eq!(reader.len(), 1000); + + assert_eq!(reader.collect_range(0, 1), vec![0]); + assert_eq!(reader.collect_range(500, 501), vec![500]); + assert_eq!(reader.collect_range(999, 1000), vec![999]); + + Ok(()) +} + +/// Test that reader doesn't see writer's uncommitted pushed data +#[test] +fn test_reader_isolation_from_pushed() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + for i in 0..50u64 { + writer.push(i); + } + writer.write()?; + + // Read-only clone + let reader = writer.read_only_clone(); + + // Writer pushes more but doesn't write + for i in 50..100u64 { + writer.push(i); + } + + // Writer sees pushed data + assert_eq!(writer.len(), 100); + assert_eq!(writer.pushed_len(), 50); + + // Read-only clone doesn't see writer's pushed + assert_eq!(reader.len(), 50); + + // Reader can't access indices 50-99 + let r = reader.reader(); + assert_eq!(r.get(49), 49); + assert_eq!(r.try_get(50), None); + + Ok(()) +} + +/// CRITICAL TEST: Verify that when reader sees updated stored_len, the data is readable. +/// This tests the memory ordering invariant: mmap write must happen-before stored_len update. +#[test] +fn test_memory_ordering_len_vs_data() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + // Write initial data + for i in 0..100u64 { + writer.push(i); + } + writer.write()?; + + let reader = writer.read_only_clone(); + + let barrier = Arc::new(Barrier::new(2)); + let stop = Arc::new(AtomicBool::new(false)); + let errors = Arc::new(AtomicUsize::new(0)); + let reads = Arc::new(AtomicUsize::new(0)); + + let reader_barrier = barrier.clone(); + let stop_clone = stop.clone(); + let errors_clone = errors.clone(); + let reads_clone = reads.clone(); + + // Reader thread: continuously check that if we see a length, we can read that data + let reader_handle = thread::spawn(move || { + // Wait for writer to be ready + reader_barrier.wait(); + + while !stop_clone.load(Ordering::Relaxed) { + let len = reader.len(); + if len > 0 { + // CRITICAL: If we see len = N, we MUST be able to read index N-1 + let last_idx = len - 1; + let r = reader.reader(); + let val = r.get(last_idx); + if val != last_idx as u64 { + eprintln!( + "ERROR: Read wrong value at {}: expected {}, got {}", + last_idx, last_idx, val + ); + errors_clone.fetch_add(1, Ordering::Relaxed); + } + reads_clone.fetch_add(1, Ordering::Relaxed); + } + // No sleep - tight loop to maximize chance of catching races + } + }); + + // Synchronize start with reader + barrier.wait(); + + // Writer thread: keep adding data + for batch in 0..100 { + for i in 0..10u64 { + let val = 100 + batch * 10 + i; + writer.push(val); + } + writer.write()?; + // Small yield to give reader a chance to run + thread::yield_now(); + } + + // Let reader run a bit more after writer is done + thread::sleep(Duration::from_millis(1)); + + stop.store(true, Ordering::Relaxed); + reader_handle.join().unwrap(); + + let error_count = errors.load(Ordering::Relaxed); + let read_count = reads.load(Ordering::Relaxed); + + println!("Completed {} reads with {} errors", read_count, error_count); + + assert_eq!(error_count, 0, "Memory ordering violation detected!"); + assert!(read_count > 0, "Should have completed at least some reads"); + + // Verify final state + assert_eq!(writer.len(), 1100); + + Ok(()) +} + +/// Test that reader always sees consistent length and can read up to that length +#[test] +fn test_length_data_consistency_stress() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + let reader = writer.read_only_clone(); + + let stop = Arc::new(AtomicBool::new(false)); + let max_len_seen = Arc::new(AtomicUsize::new(0)); + let errors = Arc::new(AtomicUsize::new(0)); + + let stop_clone = stop.clone(); + let max_len_clone = max_len_seen.clone(); + let errors_clone = errors.clone(); + + // Reader aggressively checks consistency + let reader_handle = thread::spawn(move || { + for _ in 0..1000 { + if stop_clone.load(Ordering::Relaxed) { + break; + } + + let len = reader.len(); + max_len_clone.fetch_max(len, Ordering::Relaxed); + + if len > 0 { + // Create fresh VecReader each time to pick up new stored data + let r = reader.reader(); + + // Check first, last, and a few sample indices + let indices_to_check = [0, len.saturating_sub(1), len / 2]; + + for &i in &indices_to_check { + if i >= len { + continue; + } + let val = r.get(i); + if val != i as u64 { + errors_clone.fetch_add(1, Ordering::Relaxed); + } + } + } + thread::sleep(Duration::from_micros(10)); + } + }); + + // Writer rapidly adds data + for i in 0..500u64 { + writer.push(i); + if i % 10 == 0 { + writer.write()?; + } + } + writer.write()?; + + // Let reader catch up + thread::sleep(Duration::from_millis(10)); + stop.store(true, Ordering::Relaxed); + reader_handle.join().unwrap(); + + let error_count = errors.load(Ordering::Relaxed); + assert_eq!(error_count, 0, "Consistency violation detected!"); + + // Reader should have seen at least some of the data + let max_len = max_len_seen.load(Ordering::Relaxed); + println!("Reader saw max len: {}", max_len); + assert!(max_len > 0, "Reader should have seen some data"); + + Ok(()) +} + +/// Stress test with many concurrent readers and one writer +#[test] +fn test_many_readers_one_writer() -> Result<()> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: BytesVec = BytesVec::forced_import(&db, "test_vec", version)?; + + // Initial data + for i in 0..100u64 { + writer.push(i); + } + writer.write()?; + + let num_readers = 8; + let barrier = Arc::new(Barrier::new(num_readers + 1)); + + let handles: Vec<_> = (0..num_readers) + .map(|_| { + let reader = writer.read_only_clone(); + let b = barrier.clone(); + thread::spawn(move || -> Result<()> { + b.wait(); + for _ in 0..50 { + let r = reader.reader(); + let len = r.len(); + // Verify data integrity + for i in 0..len.min(100) { + let val = r.get(i); + assert_eq!(val, i as u64); + } + thread::sleep(Duration::from_micros(10)); + } + Ok(()) + }) + }) + .collect(); + + barrier.wait(); + + // Writer keeps adding data + for batch in 0..20 { + for i in 0..50u64 { + writer.push(100 + batch * 50 + i); + } + writer.write()?; + thread::sleep(Duration::from_micros(100)); + } + + for handle in handles { + handle.join().unwrap()?; + } + + assert_eq!(writer.len(), 1100); + + Ok(()) +} + +/// Long-running realistic stress test that simulates real-world usage patterns. +/// This test: +/// - Runs for several seconds +/// - Has multiple vecs being written concurrently (like brk_computer) +/// - Has readers continuously verifying data integrity +/// - Uses batched writes without intermediate flushes +/// - Only flushes at the end of each "block" (simulating block processing) +/// +/// Run with: cargo test --features pco test_realworld_stress -- --ignored --nocapture +#[test] +#[ignore] // Run manually: takes ~10 seconds +fn test_realworld_stress() -> Result<()> { + use std::time::Instant; + + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + // Create multiple vecs (simulating different metrics in brk_computer) + let mut vec_a: BytesVec = BytesVec::forced_import(&db, "metric_a", version)?; + let mut vec_b: BytesVec = BytesVec::forced_import(&db, "metric_b", version)?; + let mut vec_c: BytesVec = BytesVec::forced_import(&db, "metric_c", version)?; + + // Write some initial data + for i in 0..1000u64 { + vec_a.push(i); + vec_b.push(i * 2); + vec_c.push(i * 3); + } + vec_a.write()?; + vec_b.write()?; + vec_c.write()?; + db.flush()?; + + // Create read-only clones (simulating web server Query clones) + let reader_a = vec_a.read_only_clone(); + let reader_b = vec_b.read_only_clone(); + let reader_c = vec_c.read_only_clone(); + + let stop = Arc::new(AtomicBool::new(false)); + let total_reads = Arc::new(AtomicUsize::new(0)); + let errors = Arc::new(AtomicUsize::new(0)); + + // Spawn multiple reader threads (simulating concurrent API requests) + let num_readers = 4; + let reader_handles: Vec<_> = (0..num_readers) + .map(|reader_id| { + let r_a = reader_a.clone(); + let r_b = reader_b.clone(); + let r_c = reader_c.clone(); + let stop = stop.clone(); + let reads = total_reads.clone(); + let errs = errors.clone(); + + thread::spawn(move || { + let mut local_reads = 0u64; + let mut local_errors = 0u64; + + while !stop.load(Ordering::Relaxed) { + // Read from all three vecs + let len_a = r_a.len(); + let len_b = r_b.len(); + let len_c = r_c.len(); + + // They should all have the same length (written together) + // Allow for small differences during concurrent writes + let max_len = len_a.max(len_b).max(len_c); + let min_len = len_a.min(len_b).min(len_c); + if max_len - min_len > 100 { + eprintln!( + "Reader {}: Large length discrepancy: a={}, b={}, c={}", + reader_id, len_a, len_b, len_c + ); + local_errors += 1; + } + + // Verify data at various positions + if min_len > 0 { + let ra = r_a.reader(); + let rb = r_b.reader(); + let rc = r_c.reader(); + + // Check first element + if ra.get(0) != 0 { + local_errors += 1; + } + + // Check last safe element + let safe_idx = min_len.saturating_sub(1); + let va = ra.get(safe_idx); + if va != safe_idx as u64 { + eprintln!( + "Reader {}: vec_a[{}] = {} (expected {})", + reader_id, safe_idx, va, safe_idx + ); + local_errors += 1; + } + if rb.get(safe_idx) != (safe_idx as u64) * 2 { + local_errors += 1; + } + if rc.get(safe_idx) != (safe_idx as u64) * 3 { + local_errors += 1; + } + + // Check a middle element + let mid_idx = min_len / 2; + if ra.get(mid_idx) != mid_idx as u64 { + local_errors += 1; + } + + local_reads += 1; + } + + // Simulate realistic request rate + thread::sleep(Duration::from_micros(100)); + } + + reads.fetch_add(local_reads as usize, Ordering::Relaxed); + errs.fetch_add(local_errors as usize, Ordering::Relaxed); + }) + }) + .collect(); + + // Writer thread: simulate processing blocks + let start = Instant::now(); + let num_blocks = 100; + let values_per_block = 50; + let mut current_idx = 1000u64; + + println!( + "Starting {} blocks of {} values each...", + num_blocks, values_per_block + ); + + for block in 0..num_blocks { + // Simulate processing a block - write to multiple vecs + for _ in 0..values_per_block { + vec_a.push(current_idx); + vec_b.push(current_idx * 2); + vec_c.push(current_idx * 3); + current_idx += 1; + } + + // Batched write: write all vecs without flushing each one + vec_a.write()?; + vec_b.write()?; + vec_c.write()?; + + // Single flush at end of block (this is the optimization we're testing) + db.flush()?; + + // Simulate some processing time between blocks + if block % 10 == 0 { + println!("Processed block {}/{}", block + 1, num_blocks); + } + thread::sleep(Duration::from_millis(10)); + } + + let write_duration = start.elapsed(); + + // Stop readers and wait for them + stop.store(true, Ordering::Relaxed); + thread::sleep(Duration::from_millis(50)); // Give readers time to notice + + for handle in reader_handles { + handle.join().unwrap(); + } + + let final_reads = total_reads.load(Ordering::Relaxed); + let final_errors = errors.load(Ordering::Relaxed); + + println!("\n=== Results ==="); + println!("Write duration: {:?}", write_duration); + println!( + "Final vec lengths: a={}, b={}, c={}", + vec_a.len(), + vec_b.len(), + vec_c.len() + ); + println!("Total reader verifications: {}", final_reads); + println!("Errors detected: {}", final_errors); + + // Verify final state + let expected_len = 1000 + (num_blocks * values_per_block) as usize; + assert_eq!(vec_a.len(), expected_len); + assert_eq!(vec_b.len(), expected_len); + assert_eq!(vec_c.len(), expected_len); + assert_eq!(final_errors, 0, "Data integrity errors detected!"); + assert!( + final_reads > 100, + "Should have completed many read verifications" + ); + + // Verify all data is correct after everything is done + println!("Verifying final data integrity..."); + let reader_a_ref = vec_a.reader(); + let reader_b_ref = vec_b.reader(); + let reader_c_ref = vec_c.reader(); + + for i in 0..expected_len { + let a = reader_a_ref.get(i); + let b = reader_b_ref.get(i); + let c = reader_c_ref.get(i); + + assert_eq!(a, i as u64, "vec_a[{}] incorrect", i); + assert_eq!(b, (i as u64) * 2, "vec_b[{}] incorrect", i); + assert_eq!(c, (i as u64) * 3, "vec_c[{}] incorrect", i); + } + + println!("All {} values verified correctly!", expected_len); + + Ok(()) +} + +/// Even longer stress test with more aggressive concurrent access +/// Run with: cargo test --features pco test_extended_stress -- --ignored --nocapture +#[test] +#[ignore] // Run manually: takes ~30 seconds +fn test_extended_stress() -> Result<()> { + use std::time::Instant; + + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + // Use compressed vecs for more realistic scenario + #[cfg(feature = "pco")] + let mut writer: PcoVec = PcoVec::forced_import(&db, "stress_vec", version)?; + #[cfg(not(feature = "pco"))] + let mut writer: BytesVec = BytesVec::forced_import(&db, "stress_vec", version)?; + + let stop = Arc::new(AtomicBool::new(false)); + let writes_completed = Arc::new(AtomicUsize::new(0)); + let reads_completed = Arc::new(AtomicUsize::new(0)); + let read_errors = Arc::new(AtomicUsize::new(0)); + + // Single reader to verify the fix works + let num_readers = 1; + let reader_handles: Vec<_> = (0..num_readers) + .map(|_| { + let reader = writer.read_only_clone(); + let stop = stop.clone(); + let reads = reads_completed.clone(); + let errs = read_errors.clone(); + + thread::spawn(move || { + let mut local_reads = 0usize; + let mut local_errors = 0usize; + + while !stop.load(Ordering::Relaxed) { + let len = reader.len(); + if len > 0 { + // Check last element - most likely to catch races + let idx = len - 1; + if let Some(v) = reader.collect_one(idx) { + if v != idx as u64 { + local_errors += 1; + } + local_reads += 1; + } else { + local_errors += 1; + } + + // Also check a random-ish index + let random_idx = (len * 7) / 11; // Pseudo-random + if random_idx < len + && let Some(v) = reader.collect_one(random_idx) + { + if v != random_idx as u64 { + local_errors += 1; + } + local_reads += 1; + } + } + // Small sleep between reads + thread::sleep(Duration::from_micros(100)); + } + + reads.fetch_add(local_reads, Ordering::Relaxed); + errs.fetch_add(local_errors, Ordering::Relaxed); + }) + }) + .collect(); + + // Writer: infrequent writes (simulates real brk_computer pattern) + let start = Instant::now(); + let target_duration = Duration::from_secs(5); + let mut current_idx = 0u64; + let mut batches = 0usize; + + println!("Running stress test for {:?}...", target_duration); + + while start.elapsed() < target_duration { + // Large batch size per write (like processing a block) + let batch_size = 100; + + for _ in 0..batch_size { + writer.push(current_idx); + current_idx += 1; + } + + writer.write()?; + writes_completed.fetch_add(1, Ordering::Relaxed); + batches += 1; + + if batches.is_multiple_of(10) { + println!("Written {} batches, {} values", batches, current_idx); + } + + // Sleep between writes - simulates ~10 minute block interval scaled down + // 100ms sleep = 50 writes over 5 seconds + thread::sleep(Duration::from_millis(100)); + + // Flush every 10 batches + if batches.is_multiple_of(10) { + db.flush()?; + } + } + + // Final flush + db.flush()?; + + let write_duration = start.elapsed(); + + // Stop readers + stop.store(true, Ordering::Relaxed); + thread::sleep(Duration::from_millis(100)); + + for handle in reader_handles { + handle.join().unwrap(); + } + + let final_writes = writes_completed.load(Ordering::Relaxed); + let final_reads = reads_completed.load(Ordering::Relaxed); + let final_errors = read_errors.load(Ordering::Relaxed); + + println!("\n=== Extended Stress Test Results ==="); + println!("Duration: {:?}", write_duration); + println!("Total values written: {}", current_idx); + println!("Write batches: {}", final_writes); + println!("Read verifications: {}", final_reads); + println!("Errors: {}", final_errors); + println!( + "Reads per second: {:.0}", + final_reads as f64 / write_duration.as_secs_f64() + ); + + assert_eq!(writer.len(), current_idx as usize); + assert_eq!(final_errors, 0, "Data integrity errors detected!"); + assert!(final_reads > 1000, "Should have many read verifications"); + + // Spot-check final data + println!("Spot-checking final data..."); + for i in [ + 0, + 100, + 1000, + current_idx as usize / 2, + current_idx as usize - 1, + ] { + if i < writer.len() { + let v = writer.collect_one(i).unwrap(); + assert_eq!(v, i as u64, "Value at {} incorrect", i); + } + } + println!("Spot-check passed!"); + + Ok(()) +} + +/// BytesVec version of extended stress test - runs without compression overhead +/// This allows for a true tight loop test without reader starvation issues. +/// +/// Run with: cargo test test_extended_stress_bytes -- --ignored --nocapture +#[test] +#[ignore] // Run manually +fn test_extended_stress_bytes() -> Result<()> { + use std::time::Instant; + + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: BytesVec = + BytesVec::forced_import(&db, "stress_vec_bytes", version)?; + + let stop = Arc::new(AtomicBool::new(false)); + let writes_completed = Arc::new(AtomicUsize::new(0)); + let reads_completed = Arc::new(AtomicUsize::new(0)); + let read_errors = Arc::new(AtomicUsize::new(0)); + + // More aggressive readers - BytesVec is fast enough for tight loops + let num_readers = 8; + let reader_handles: Vec<_> = (0..num_readers) + .map(|_| { + let reader = writer.read_only_clone(); + let stop = stop.clone(); + let reads = reads_completed.clone(); + let errs = read_errors.clone(); + + thread::spawn(move || { + let mut local_reads = 0usize; + let mut local_errors = 0usize; + + while !stop.load(Ordering::Relaxed) { + let r = reader.reader(); + let len = r.len(); + if len > 0 { + // Check last element - most likely to catch races + let idx = len - 1; + let v = r.get(idx); + if v != idx as u64 { + local_errors += 1; + } + local_reads += 1; + + // Also check a random-ish index + let random_idx = (len * 7) / 11; + if random_idx < len { + let v = r.get(random_idx); + if v != random_idx as u64 { + local_errors += 1; + } + local_reads += 1; + } + } + // Very tight loop for BytesVec - no sleep needed + } + + reads.fetch_add(local_reads, Ordering::Relaxed); + errs.fetch_add(local_errors, Ordering::Relaxed); + }) + }) + .collect(); + + // Writer: rapid writes with varying batch sizes + let start = Instant::now(); + let target_duration = Duration::from_secs(5); + let mut current_idx = 0u64; + let mut batches = 0usize; + + println!("Running BytesVec stress test for {:?}...", target_duration); + + while start.elapsed() < target_duration { + // Varying batch sizes + let batch_size = 10 + (batches % 50); + + for _ in 0..batch_size { + writer.push(current_idx); + current_idx += 1; + } + + writer.write()?; + writes_completed.fetch_add(1, Ordering::Relaxed); + batches += 1; + + // Occasionally flush (simulating block boundaries) + if batches.is_multiple_of(10) { + db.flush()?; + } + } + + // Final flush + db.flush()?; + + let write_duration = start.elapsed(); + + // Stop readers + stop.store(true, Ordering::Relaxed); + thread::sleep(Duration::from_millis(100)); + + for handle in reader_handles { + handle.join().unwrap(); + } + + let final_writes = writes_completed.load(Ordering::Relaxed); + let final_reads = reads_completed.load(Ordering::Relaxed); + let final_errors = read_errors.load(Ordering::Relaxed); + + println!("\n=== BytesVec Extended Stress Test Results ==="); + println!("Duration: {:?}", write_duration); + println!("Total values written: {}", current_idx); + println!("Write batches: {}", final_writes); + println!("Read verifications: {}", final_reads); + println!("Errors: {}", final_errors); + println!( + "Reads per second: {:.0}", + final_reads as f64 / write_duration.as_secs_f64() + ); + + assert_eq!(writer.len(), current_idx as usize); + assert_eq!(final_errors, 0, "Data integrity errors detected!"); + assert!( + final_reads > 10000, + "Should have many read verifications with tight loop" + ); + + // Spot-check final data + println!("Spot-checking final data..."); + let reader_ref = writer.reader(); + for i in [ + 0, + 100, + 1000, + current_idx as usize / 2, + current_idx as usize - 1, + ] { + if i < writer.len() { + let v = reader_ref.get(i); + assert_eq!(v, i as u64, "Value at {} incorrect", i); + } + } + println!("Spot-check passed!"); + + Ok(()) +} diff --git a/crates/vecdb/tests/consistency.rs b/crates/vecdb/tests/consistency.rs new file mode 100644 index 000000000..2e79985fc --- /dev/null +++ b/crates/vecdb/tests/consistency.rs @@ -0,0 +1,180 @@ +//! Generic consistency tests for all vec types. + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{AnyStoredVec, EagerVec, ImportableVec, ReadableVec, StoredVec, Version, WritableVec}; + +// ============================================================================ +// Generic Test Functions +// ============================================================================ + +/// Generic test function for mmap write/file read consistency +fn run_mmap_write_file_read_consistency() +where + V: StoredVec, +{ + let temp_dir = TempDir::new().unwrap(); + let db = Database::open(&temp_dir.path().join("test.db")).unwrap(); + + // Create a vec (which uses mmap for writes) + let mut vec: EagerVec = EagerVec::forced_import(&db, "test_vec", Version::ONE).unwrap(); + + // Write some data + for i in 0..1000usize { + vec.checked_push_at(i, i as u64 * 100).unwrap(); + } + + // Flush the vec (writes to mmap) + vec.flush().unwrap(); + + println!("After flush, checking data consistency..."); + + // Check if collected data matches what was written + let collected = vec.collect(); + (0..1000usize).for_each(|i| { + let value = collected[i]; + let expected = i as u64 * 100; + + if value != expected { + panic!( + "Inconsistency detected at index {}: got {}, expected {}", + i, value, expected + ); + } + }); + + println!("Test passed! All values consistent."); +} + +/// Generic test function for immediate read after write +fn run_immediate_read_after_write() +where + V: StoredVec, +{ + let temp_dir = TempDir::new().unwrap(); + let db = Database::open(&temp_dir.path().join("test2.db")).unwrap(); + + let mut vec: EagerVec = EagerVec::forced_import(&db, "test_vec", Version::ONE).unwrap(); + + // Write, flush, read immediately (mimics the txinindex -> txindex pattern) + for batch in 0..10 { + let start = batch * 100; + + // Write batch + for i in 0..100usize { + vec.checked_push_at(start + i, (start + i) as u64 * 100) + .unwrap(); + } + + // Flush + vec.flush().unwrap(); + + // Immediately read back using collect_range + for i in 0..100usize { + let idx = start + i; + let value = vec.collect_one(idx).unwrap(); + let expected = (start + i) as u64 * 100; + + if value != expected { + panic!( + "Batch {} inconsistency at index {}: got {}, expected {}", + batch, idx, value, expected + ); + } + } + } + + println!("Immediate read test passed!"); +} + +// ============================================================================ +// Test instantiation for BytesVec (no feature flag needed) +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn mmap_write_file_read_consistency() { + run_mmap_write_file_read_consistency::(); + } + + #[test] + fn immediate_read_after_write() { + run_immediate_read_after_write::(); + } +} + +// ============================================================================ +// Test instantiation for feature-gated vec types +// ============================================================================ + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn mmap_write_file_read_consistency() { + run_mmap_write_file_read_consistency::(); + } + + #[test] + fn immediate_read_after_write() { + run_immediate_read_after_write::(); + } +} + +#[cfg(feature = "pco")] +mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn mmap_write_file_read_consistency() { + run_mmap_write_file_read_consistency::(); + } + + #[test] + fn immediate_read_after_write() { + run_immediate_read_after_write::(); + } +} + +#[cfg(feature = "lz4")] +mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn mmap_write_file_read_consistency() { + run_mmap_write_file_read_consistency::(); + } + + #[test] + fn immediate_read_after_write() { + run_immediate_read_after_write::(); + } +} + +#[cfg(feature = "zstd")] +mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn mmap_write_file_read_consistency() { + run_mmap_write_file_read_consistency::(); + } + + #[test] + fn immediate_read_after_write() { + run_immediate_read_after_write::(); + } +} diff --git a/crates/vecdb/tests/cursor.rs b/crates/vecdb/tests/cursor.rs new file mode 100644 index 000000000..d53a86880 --- /dev/null +++ b/crates/vecdb/tests/cursor.rs @@ -0,0 +1,84 @@ +#![cfg(feature = "pco")] + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, ImportableVec, PcoVec, ReadableVec, Result, StoredVec, Version, WritableVec, +}; + +#[test] +fn pco_u8_cursor_crosses_page_and_chunk_boundaries() -> Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + let mut vec = PcoVec::::import(&db, "u8", Version::ONE)?; + + for index in 0..20_000 { + vec.push((index % 251) as u8); + } + vec.write()?; + let read_only = vec.read_only_clone(); + + for index in 20_000..20_010 { + vec.push((index % 251) as u8); + } + + let mut cursor = vec.cursor(); + for index in [0, 4_095, 4_096, 16_383, 16_384, 19_999, 20_009] { + assert_eq!(cursor.get(index), Some((index % 251) as u8)); + assert_eq!(cursor.position(), 0); + } + assert_eq!(cursor.get(20_010), None); + + cursor.advance(16_380); + for index in 16_380..16_390 { + assert_eq!(cursor.next(), Some((index % 251) as u8)); + } + + let mut tail = Vec::new(); + cursor.advance(20_000 - cursor.position()); + cursor.for_each(usize::MAX, |value| tail.push(value)); + assert_eq!( + tail, + (20_000..20_010) + .map(|index| (index % 251) as u8) + .collect::>() + ); + + let mut read_only_cursor = read_only.cursor(); + read_only_cursor.advance(16_380); + let values = read_only_cursor.fold(20, Vec::new(), |mut values, value| { + values.push(value); + values + }); + assert_eq!( + values, + (16_380..16_400) + .map(|index| (index % 251) as u8) + .collect::>() + ); + assert_eq!(read_only_cursor.get(20_000), None); + + Ok(()) +} + +#[test] +fn pco_u64_cursor_crosses_two_page_chunk_boundary() -> Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + let mut vec = PcoVec::::import(&db, "u64", Version::ONE)?; + + for index in 0..5_000 { + vec.push((index as u64).wrapping_mul(37)); + } + vec.write()?; + + let mut cursor = vec.cursor(); + cursor.advance(4_090); + for index in 4_090..4_110 { + assert_eq!(cursor.next(), Some((index as u64).wrapping_mul(37))); + } + assert_eq!(cursor.get(4_999), Some(4_999_u64 * 37)); + assert_eq!(cursor.get(5_000), None); + + Ok(()) +} diff --git a/crates/vecdb/tests/cursor_chunk_size.rs b/crates/vecdb/tests/cursor_chunk_size.rs new file mode 100644 index 000000000..265da5da2 --- /dev/null +++ b/crates/vecdb/tests/cursor_chunk_size.rs @@ -0,0 +1,29 @@ +#![cfg(feature = "pco")] + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ + BytesVec, ImportableVec, PcoVec, READ_CHUNK_SIZE, ReadableVec, Result, StoredVec, Version, +}; + +#[test] +fn compressed_cursor_chunks_are_page_aligned() -> Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let raw = BytesVec::::import(&db, "raw", Version::ONE)?; + assert_eq!(raw.cursor_chunk_size(), READ_CHUNK_SIZE); + + let pco_u8 = PcoVec::::import(&db, "pco_u8", Version::ONE)?; + assert_eq!(pco_u8.cursor_chunk_size(), 16 * 1024); + assert_eq!(pco_u8.read_only_clone().cursor_chunk_size(), 16 * 1024); + + let pco_u64 = PcoVec::::import(&db, "pco_u64", Version::ONE)?; + assert_eq!(pco_u64.cursor_chunk_size(), READ_CHUNK_SIZE); + assert_eq!( + pco_u64.read_only_clone().cursor_chunk_size(), + READ_CHUNK_SIZE + ); + + Ok(()) +} diff --git a/crates/vecdb/tests/integrity.rs b/crates/vecdb/tests/integrity.rs new file mode 100644 index 000000000..175f0ebf3 --- /dev/null +++ b/crates/vecdb/tests/integrity.rs @@ -0,0 +1,518 @@ +//! Data integrity tests for RawVec types (BytesVec and ZeroCopyVec). +//! +//! Tests verify that rollback + flush + reopen preserves data correctly. + +use rawdb::Database; +use sha2::{Digest, Sha256}; +use std::fs; +use std::ops::{Deref, DerefMut}; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, ImportOptions, ImportableVec, RawStrategy, ReadWriteRawVec, ReadableVec, Result, + Stamp, StoredVec, VecReader, Version, WritableVec, +}; + +// ============================================================================ +// Traits for Raw Vec Types (supporting holes/updates) +// ============================================================================ + +pub trait IntegrityVec: StoredVec + DerefMut + ImportableVec + Sized +where + Self::Target: IntegrityOps, +{ + fn import_with_changes<'a>( + db: &'a Database, + name: &'a str, + changes: u16, + ) -> Result<(Self, ImportOptions<'a>)>; +} + +pub trait IntegrityOps { + type Reader; + + fn update(&mut self, index: usize, value: u32) -> Result<()>; + fn take(&mut self, index: usize) -> Option; + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()>; + fn stamp(&self) -> Stamp; + fn collect(&self) -> Vec; + fn collect_holed(&self) -> Vec>; + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option; + fn reader(&self) -> Self::Reader; +} + +// Generic implementations for any ReadWriteRawVec strategy +impl IntegrityOps for ReadWriteRawVec +where + S: RawStrategy, +{ + type Reader = VecReader; + + fn update(&mut self, index: usize, value: u32) -> Result<()> { + ReadWriteRawVec::update(self, index, value) + } + + fn take(&mut self, index: usize) -> Option { + let reader = self.reader(); + let result = ReadWriteRawVec::take(self, index, &reader); + drop(reader); + result + } + + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + WritableVec::stamped_write_with_changes(self, stamp) + } + + fn stamp(&self) -> Stamp { + AnyStoredVec::stamp(self) + } + + fn collect(&self) -> Vec { + ReadableVec::collect(self) + } + + fn collect_holed(&self) -> Vec> { + ReadWriteRawVec::collect_holed(self) + } + + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option { + ReadWriteRawVec::get_with_reader(self, index, reader) + } + + fn reader(&self) -> Self::Reader { + ReadWriteRawVec::reader(self) + } +} + +/// Helper to create a temporary test database +pub fn setup_test_db() -> Result<(Database, TempDir)> { + let temp_dir = TempDir::new()?; + let db = Database::open(temp_dir.path())?; + Ok((db, temp_dir)) +} + +/// Compute SHA-256 hash of the vecdb data file and regions directory +/// Only hashes data (file) and regions/*, ignoring changes directory +fn compute_directory_hash(dir: &Path) -> Result> { + let mut hasher = Sha256::new(); + + // Collect all files in sorted order for deterministic hashing + let mut files: Vec = Vec::new(); + + // Hash the data file if it exists + let data_file = dir.join("data"); + if data_file.exists() && data_file.is_file() { + files.push(data_file); + } + + // Hash files in the regions directory, excluding changes subdirectory + let regions_dir = dir.join("regions"); + if regions_dir.exists() { + fn collect_files(dir: &Path, files: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.components().any(|c| c.as_os_str() == "changes") { + continue; + } + if path.is_dir() { + collect_files(&path, files); + } else if path.is_file() { + files.push(path); + } + } + } + collect_files(®ions_dir, &mut files); + } + + files.sort(); + + // Hash each file's relative path and contents + for file_path in &files { + // Hash the relative path + if let Ok(rel_path) = file_path.strip_prefix(dir) { + hasher.update(rel_path.to_string_lossy().as_bytes()); + } + + // Hash the file contents + let contents = fs::read(file_path)?; + hasher.update(&contents); + } + + let hash = hasher.finalize(); + Ok(hash.iter().map(|b| format!("{:02x}", b)).collect()) +} + +/// Generic test function for data integrity with rollback, flush, and reopen +/// +/// This test verifies that after rollback + flush + close + reopen: +/// 1. Data can be correctly read back using individual gets +/// 2. Data can be correctly read back using iterators +/// 3. Redo operations produce the same readable state +fn run_data_integrity_rollback_flush_reopen() -> Result<(), Box> +where + V: IntegrityVec + WritableVec, + ::Target: IntegrityOps, +{ + println!("=== Data Integrity Test: Rollback + Flush + Reopen ===\n"); + println!("This test verifies:"); + println!(" • Rollback + flush + reopen preserves data correctly"); + println!(" • Data readable via both gets and iterators"); + println!(" • File hashes track physical layout differences\n"); + + // Create database + let (database, temp) = setup_test_db()?; + let test_path = temp.path(); + + let (mut vec, _options) = V::import_with_changes(&database, "vec", 10)?; + println!("✓ Created vecdb"); + + // Step 1: Do initial work + println!("\n--- Phase 1: Initial work ---"); + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + println!("✓ Added values [0, 1, 2, 3, 4] and flushed (stamp 1)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Step 2: More work and flush + println!("\n--- Phase 2: More work ---"); + for i in 5..10 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + println!("✓ Added values [5, 6, 7, 8, 9] and flushed (stamp 2)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Step 3: Checkpoint 1 - Save hash and data state + println!("\n--- Checkpoint 1 ---"); + let checkpoint1_data = vec.deref_mut().collect_holed(); + let checkpoint1_stamp = vec.deref_mut().stamp(); + let checkpoint1_hash = compute_directory_hash(test_path)?; + println!("✓ Saved checkpoint1 at stamp {:?}", checkpoint1_stamp); + println!(" Data: {:?}", checkpoint1_data); + println!(" Length: {}", vec.len()); + println!(" File hash: {}", checkpoint1_hash); + + // Step 4-6: Three more operations with flush + println!("\n--- Phase 3: Three more operations ---"); + + // Operation 1: Update some values + vec.deref_mut().update(2, 100)?; + vec.deref_mut().update(7, 200)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + println!("✓ Operation 1: Updated index 2→100, 7→200 (stamp 3)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Operation 2: Add more values + vec.push(20); + vec.push(21); + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; + println!("✓ Operation 2: Added values [20, 21] (stamp 4)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Operation 3: Create a hole and add value + let _ = vec.deref_mut().take(5); + vec.push(30); + vec.deref_mut().stamped_write_with_changes(Stamp::new(5))?; + println!("✓ Operation 3: Removed index 5, added value 30 (stamp 5)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + println!(" Data with holes: {:?}", vec.deref_mut().collect_holed()); + + // Step 7: Checkpoint 2 - Save hash and data state + println!("\n--- Checkpoint 2 ---"); + let checkpoint2_data = vec.deref_mut().collect_holed(); + let checkpoint2_stamp = vec.deref_mut().stamp(); + let checkpoint2_hash = compute_directory_hash(test_path)?; + println!("✓ Saved checkpoint2 at stamp {:?}", checkpoint2_stamp); + println!(" Data: {:?}", checkpoint2_data); + println!(" Length: {}", vec.len()); + println!(" File hash: {}", checkpoint2_hash); + + // Step 8: Undo last 3 operations + println!("\n--- Phase 4: Undo last 3 operations ---"); + vec.rollback()?; + println!("✓ Rollback 1: Now at stamp {:?}", vec.deref_mut().stamp()); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + vec.rollback()?; + println!("✓ Rollback 2: Now at stamp {:?}", vec.deref_mut().stamp()); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + vec.rollback()?; + println!("✓ Rollback 3: Now at stamp {:?}", vec.deref_mut().stamp()); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Step 9: Verify in-memory data matches checkpoint1 + println!("\n--- Verification 1: After undo (in-memory) ---"); + let after_undo_data = vec.deref_mut().collect_holed(); + let after_undo_stamp = vec.deref_mut().stamp(); + println!("In-memory state after rollback:"); + println!( + " Stamp: {:?} (expected: {:?})", + after_undo_stamp, checkpoint1_stamp + ); + println!(" Data: {:?}", after_undo_data); + + assert_eq!( + after_undo_stamp, checkpoint1_stamp, + "Stamp mismatch after undo!" + ); + assert_eq!( + after_undo_data, checkpoint1_data, + "In-memory data mismatch after undo!" + ); + println!("✓ PASS: In-memory data matches checkpoint1 after undo"); + + // Flush and close + println!("\n--- Step 10: Flush, close, and reopen ---"); + vec.deref_mut() + .stamped_write_with_changes(checkpoint1_stamp)?; + let after_flush_hash = compute_directory_hash(test_path)?; + println!("✓ Flushed to disk"); + println!(" File hash: {}", after_flush_hash); + if after_flush_hash != checkpoint1_hash { + println!( + " Note: Hash differs from checkpoint1 ({}) due to region allocation bug", + checkpoint1_hash + ); + } + + // Drop the vec to close files + drop(vec); + println!("✓ Closed vecdb"); + + // Reopen the database + let (mut vec, _options) = V::import_with_changes(&database, "vec", 10)?; + println!("✓ Reopened vecdb"); + println!(" Stamp after reopen: {:?}", vec.deref_mut().stamp()); + println!(" Length after reopen: {}", vec.len()); + + // Verify using individual gets + println!("\n--- Verification 2: After reopen (using gets) ---"); + let reader = vec.deref_mut().reader(); + let mut data_via_gets = Vec::new(); + for i in 0..vec.len() { + let value = vec.deref_mut().get_with_reader(i, &reader); + data_via_gets.push(value); + } + drop(reader); + + println!("Data read via gets: {:?}", data_via_gets); + assert_eq!( + data_via_gets, checkpoint1_data, + "Data mismatch reading via gets after reopen!" + ); + println!("✓ PASS: Data correct when reading via gets"); + + // Verify using iterator + println!("\n--- Verification 3: After reopen (using iterator) ---"); + let data_via_iter = vec.deref_mut().collect_holed(); + println!("Data read via iterator: {:?}", data_via_iter); + assert_eq!( + data_via_iter, checkpoint1_data, + "Data mismatch reading via iterator after reopen!" + ); + println!("✓ PASS: Data correct when reading via iterator"); + + // Also test the clean iterator (non-holed) + let data_via_clean_iter: Vec = vec.deref_mut().collect(); + let expected_clean: Vec = checkpoint1_data.iter().filter_map(|x| *x).collect(); + println!("Data via clean iterator: {:?}", data_via_clean_iter); + assert_eq!( + data_via_clean_iter, expected_clean, + "Data mismatch via clean iterator!" + ); + println!("✓ PASS: Data correct when reading via clean iterator"); + + println!("\n✓ ALL VERIFICATION PASSED: Rollback + flush + reopen preserved data correctly!"); + + // Step 10: Redo the same 3 operations + println!("\n--- Phase 5: Redo same 3 operations ---"); + + // Redo Operation 1: Update same values + vec.deref_mut().update(2, 100)?; + vec.deref_mut().update(7, 200)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + println!("✓ Redo operation 1: Updated index 2→100, 7→200 (stamp 3)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Redo Operation 2: Add same values + vec.push(20); + vec.push(21); + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; + println!("✓ Redo operation 2: Added values [20, 21] (stamp 4)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + + // Redo Operation 3: Create hole and add value + let _ = vec.deref_mut().take(5); + vec.push(30); + vec.deref_mut().stamped_write_with_changes(Stamp::new(5))?; + println!("✓ Redo operation 3: Removed index 5, added value 30 (stamp 5)"); + println!(" Current data: {:?}", vec.deref_mut().collect()); + println!(" Data with holes: {:?}", vec.deref_mut().collect_holed()); + + // Step 11: Verify in-memory data matches checkpoint2 + println!("\n--- Verification 4: After redo (in-memory) ---"); + let after_redo_data = vec.deref_mut().collect_holed(); + let after_redo_stamp = vec.deref_mut().stamp(); + println!("In-memory state after redo:"); + println!( + " Stamp: {:?} (expected: {:?})", + after_redo_stamp, checkpoint2_stamp + ); + println!(" Data: {:?}", after_redo_data); + + assert_eq!( + after_redo_stamp, checkpoint2_stamp, + "Stamp mismatch after redo!" + ); + assert_eq!( + after_redo_data, checkpoint2_data, + "In-memory data mismatch after redo!" + ); + println!("✓ PASS: In-memory data matches checkpoint2 after redo"); + + // Flush and close + println!("\n--- Step 12: Flush, close, and reopen (after redo) ---"); + vec.deref_mut() + .stamped_write_with_changes(checkpoint2_stamp)?; + let after_redo_flush_hash = compute_directory_hash(test_path)?; + println!("✓ Flushed to disk"); + println!(" File hash: {}", after_redo_flush_hash); + if after_redo_flush_hash == checkpoint2_hash { + println!(" ✓ Hash matches checkpoint2 (operations were deterministic)"); + } else { + println!( + " Note: Hash differs from checkpoint2 ({})", + checkpoint2_hash + ); + } + + // Drop and reopen + drop(vec); + println!("✓ Closed vecdb"); + + let (mut vec, _options) = V::import_with_changes(&database, "vec", 10)?; + println!("✓ Reopened vecdb"); + println!(" Stamp after reopen: {:?}", vec.deref_mut().stamp()); + println!(" Length after reopen: {}", vec.len()); + + // Verify using individual gets + println!("\n--- Verification 5: After reopen (using gets) ---"); + let reader = vec.deref_mut().reader(); + let mut data_via_gets = Vec::new(); + for i in 0..vec.len() { + let value = vec.deref_mut().get_with_reader(i, &reader); + data_via_gets.push(value); + } + drop(reader); + + println!("Data read via gets: {:?}", data_via_gets); + assert_eq!( + data_via_gets, checkpoint2_data, + "Data mismatch reading via gets after reopen!" + ); + println!("✓ PASS: Data correct when reading via gets"); + + // Verify using iterator + println!("\n--- Verification 6: After reopen (using iterator) ---"); + let data_via_iter = vec.deref_mut().collect_holed(); + println!("Data read via iterator: {:?}", data_via_iter); + assert_eq!( + data_via_iter, checkpoint2_data, + "Data mismatch reading via iterator after reopen!" + ); + println!("✓ PASS: Data correct when reading via iterator"); + + // Also test the clean iterator (non-holed) + let data_via_clean_iter: Vec = vec.deref_mut().collect(); + let expected_clean: Vec = checkpoint2_data.iter().filter_map(|x| *x).collect(); + println!("Data via clean iterator: {:?}", data_via_clean_iter); + assert_eq!( + data_via_clean_iter, expected_clean, + "Data mismatch via clean iterator!" + ); + println!("✓ PASS: Data correct when reading via clean iterator"); + + println!("\n✓ ALL VERIFICATION PASSED: Redo + flush + reopen preserved data correctly!"); + + println!("\n=== Test Results ==="); + println!("\nData integrity:"); + println!("✓ Rollback + flush + reopen: Data correctly preserved"); + println!("✓ Redo + flush + reopen: Data correctly preserved"); + println!("✓ Gets work correctly after reopen"); + println!("✓ Iterators (holed and clean) work correctly after reopen"); + println!("\nFile layout:"); + println!(" Checkpoint1 hash: {}", checkpoint1_hash); + println!(" After rollback: {}", after_flush_hash); + if after_flush_hash != checkpoint1_hash { + println!(" ⚠ Hashes differ due to region allocation bug (regions not rolled back)"); + } + println!(" Checkpoint2 hash: {}", checkpoint2_hash); + println!(" After redo: {}", after_redo_flush_hash); + if after_redo_flush_hash == checkpoint2_hash { + println!(" ✓ Hashes match (redo is deterministic)"); + } + println!("\n✓ ALL TESTS PASSED - Data integrity maintained across rollback/redo cycle!"); + + Ok(()) +} + +// ============================================================================ +// Test instantiation for BytesVec (no feature flag needed) +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + + impl IntegrityVec for BytesVec { + fn import_with_changes<'a>( + db: &'a Database, + name: &'a str, + changes: u16, + ) -> Result<(Self, ImportOptions<'a>)> { + let mut options: ImportOptions = (db, name, Version::TWO).into(); + options = options.with_saved_stamped_changes(changes); + let vec = Self::forced_import_with(options)?; + Ok((vec, options)) + } + } + + #[test] + fn data_integrity_rollback_flush_reopen() -> Result<(), Box> { + run_data_integrity_rollback_flush_reopen::>() + } +} + +// ============================================================================ +// Test instantiation for ZeroCopyVec (with zerocopy feature) +// ============================================================================ + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + + impl IntegrityVec for ZeroCopyVec { + fn import_with_changes<'a>( + db: &'a Database, + name: &'a str, + changes: u16, + ) -> Result<(Self, ImportOptions<'a>)> { + let mut options: ImportOptions = (db, name, Version::TWO).into(); + options = options.with_saved_stamped_changes(changes); + let vec = Self::forced_import_with(options)?; + Ok((vec, options)) + } + } + + #[test] + fn data_integrity_rollback_flush_reopen() -> Result<(), Box> { + run_data_integrity_rollback_flush_reopen::>() + } +} diff --git a/crates/vecdb/tests/iterators.rs b/crates/vecdb/tests/iterators.rs new file mode 100644 index 000000000..e9b9e4a08 --- /dev/null +++ b/crates/vecdb/tests/iterators.rs @@ -0,0 +1,1795 @@ +//! Generic iterator tests for all vec types. +//! +//! These tests run against any type implementing `StoredVec`, ensuring +//! consistent iterator behavior across BytesVec, ZeroCopyVec, PcoVec, LZ4Vec, and ZstdVec. + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{Result, StoredVec, Version}; + +// ============================================================================ +// Test Setup Helpers +// ============================================================================ + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +// ============================================================================ +// Generic Clean Iterator Tests +// ============================================================================ + +mod clean_iter { + use super::*; + + fn run_basic() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 100); + assert_eq!(collected[0], 0); + assert_eq!(collected[99], 99); + Ok(()) + } + + fn run_nth() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + // Test point reads at specific indices + assert_eq!(vec.collect_first().unwrap(), 0); + assert_eq!(vec.collect_one(10).unwrap(), 10); + assert_eq!(vec.collect_one(11).unwrap(), 11); + Ok(()) + } + + fn run_skip() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected = vec.collect_range(50, 100); + assert_eq!(collected.len(), 50); + assert_eq!(collected[0], 50); + Ok(()) + } + + fn run_take() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected = vec.collect_range(0, 25); + assert_eq!(collected.len(), 25); + assert_eq!(collected[24], 24); + Ok(()) + } + + fn run_set_position() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected = vec.collect_range(50, 52); + assert_eq!(collected[0], 50); + assert_eq!(collected[1], 51); + Ok(()) + } + + fn run_set_end() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected = vec.collect_range(0, 50); + assert_eq!(collected.len(), 50); + Ok(()) + } + + fn run_last() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + assert_eq!(vec.collect_last().unwrap(), 99); + Ok(()) + } + + fn run_last_empty() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let vec = V::forced_import(&db, "test", Version::ONE)?; + + assert_eq!(vec.len(), 0); + assert!(vec.collect().is_empty()); + Ok(()) + } + + fn run_exact_size() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + assert_eq!(vec.len(), 100); + assert_eq!(vec.collect_range(0, 100).len(), 100); + assert_eq!(vec.collect_range(1, 100).len(), 99); + Ok(()) + } + + fn run_buffer_crossing() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10000 { + vec.push(i); + } + vec.write()?; + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 10000); + + for (i, &val) in collected.iter().enumerate() { + assert_eq!(val, i as i32); + } + Ok(()) + } + + fn run_multiple_skip_take() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..1000 { + vec.push(i); + } + vec.write()?; + + // skip(100).take(200).skip(50).take(100) => collect_range(150, 250) + let collected = vec.collect_range(150, 250); + + assert_eq!(collected.len(), 100); + assert_eq!(collected[0], 150); + assert_eq!(collected[99], 249); + Ok(()) + } + + fn run_set_position_multiple() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..1000 { + vec.push(i); + } + vec.write()?; + + // Random access reads at various positions + assert_eq!(vec.collect_one(100).unwrap(), 100); + assert_eq!(vec.collect_one(500).unwrap(), 500); + assert_eq!(vec.collect_one(50).unwrap(), 50); + Ok(()) + } + + fn run_nth_beyond_end() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10 { + vec.push(i); + } + vec.write()?; + + // Reading beyond end returns empty + let collected = vec.collect_range(10, 10); + assert!(collected.is_empty()); + Ok(()) + } + + fn run_skip_all() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected = vec.collect_range(100, 100); + assert!(collected.is_empty()); + Ok(()) + } + + fn run_take_zero() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected = vec.collect_range(0, 0); + assert_eq!(collected.len(), 0); + Ok(()) + } + + fn run_size_hint_consistency() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + // Test that collect_range returns correct sizes for progressive ranges + for i in 0..100 { + let remaining = vec.collect_range(i, 100); + assert_eq!(remaining.len(), 100 - i); + } + Ok(()) + } + + // ============================================================================ + // Test instantiation for each vec type + // ============================================================================ + + mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } + + #[cfg(feature = "zerocopy")] + mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } + + #[cfg(feature = "pco")] + mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } + + #[cfg(feature = "lz4")] + mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } + + #[cfg(feature = "zstd")] + mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } + + // ============================================================================ + // EagerVec Tests (wrapping different underlying vec types) + // ============================================================================ + + #[cfg(feature = "zerocopy")] + mod eager_zerocopy { + use super::*; + use vecdb::{EagerVec, ZeroCopyVec}; + type V = EagerVec>; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } + + #[cfg(feature = "pco")] + mod eager_pco { + use super::*; + use vecdb::{EagerVec, PcoVec}; + type V = EagerVec>; + + #[test] + fn basic() -> Result<()> { + run_basic::() + } + #[test] + fn nth() -> Result<()> { + run_nth::() + } + #[test] + fn skip() -> Result<()> { + run_skip::() + } + #[test] + fn take() -> Result<()> { + run_take::() + } + #[test] + fn set_position() -> Result<()> { + run_set_position::() + } + #[test] + fn set_end() -> Result<()> { + run_set_end::() + } + #[test] + fn last() -> Result<()> { + run_last::() + } + #[test] + fn last_empty() -> Result<()> { + run_last_empty::() + } + #[test] + fn exact_size() -> Result<()> { + run_exact_size::() + } + #[test] + fn buffer_crossing() -> Result<()> { + run_buffer_crossing::() + } + #[test] + fn multiple_skip_take() -> Result<()> { + run_multiple_skip_take::() + } + #[test] + fn set_position_multiple() -> Result<()> { + run_set_position_multiple::() + } + #[test] + fn nth_beyond_end() -> Result<()> { + run_nth_beyond_end::() + } + #[test] + fn skip_all() -> Result<()> { + run_skip_all::() + } + #[test] + fn take_zero() -> Result<()> { + run_take_zero::() + } + #[test] + fn size_hint_consistency() -> Result<()> { + run_size_hint_consistency::() + } + } +} + +// ============================================================================ +// Generic Dirty Iterator Tests (stored + pushed data) +// ============================================================================ + +mod dirty_iter { + use super::*; + + fn run_only_stored() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 100); + assert_eq!(collected[0], 0); + assert_eq!(collected[99], 99); + Ok(()) + } + + fn run_only_pushed() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + // Don't flush + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 50); + assert_eq!(collected[0], 0); + assert_eq!(collected[49], 49); + Ok(()) + } + + fn run_stored_and_pushed() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..100 { + vec.push(i); + } + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 100); + + for (i, &val) in collected.iter().enumerate() { + assert_eq!(val, i as i32); + } + Ok(()) + } + + fn run_skip_across_boundary() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..100 { + vec.push(i); + } + + let collected = vec.collect_range(40, 100); + assert_eq!(collected.len(), 60); + assert_eq!(collected[0], 40); + assert_eq!(collected[59], 99); + Ok(()) + } + + fn run_take_across_boundary() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..100 { + vec.push(i); + } + + let collected = vec.collect_range(40, 60); + assert_eq!(collected.len(), 20); + assert_eq!(collected[0], 40); + assert_eq!(collected[19], 59); + Ok(()) + } + + fn run_nth_across_boundary() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..100 { + vec.push(i); + } + + // Test reading across stored/pushed boundary + let vals = vec.collect_range(45, 52); + assert_eq!(vals[0], 45); // In stored + assert_eq!(vals[1], 46); // In stored + assert_eq!(vals[4], 49); // In stored + assert_eq!(vals[5], 50); // In pushed + assert_eq!(vals[6], 51); // In pushed + Ok(()) + } + + fn run_set_position_to_pushed() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..100 { + vec.push(i); + } + + let vals = vec.collect_range(75, 77); + assert_eq!(vals[0], 75); + assert_eq!(vals[1], 76); + Ok(()) + } + + fn run_last_in_pushed() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..100 { + vec.push(i); + } + + assert_eq!(vec.collect_last().unwrap(), 99); + Ok(()) + } + + fn run_last_in_stored() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..100 { + vec.push(i); + } + vec.write()?; + + assert_eq!(vec.collect_last().unwrap(), 99); + Ok(()) + } + + fn run_exact_size_with_pushed() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..50 { + vec.push(i); + } + vec.write()?; + + for i in 50..75 { + vec.push(i); + } + + assert_eq!(vec.len(), 75); + assert_eq!(vec.collect_range(0, 75).len(), 75); + assert_eq!(vec.collect_range(1, 75).len(), 74); + assert_eq!(vec.collect_range(51, 75).len(), 24); // Cross boundary + Ok(()) + } + + fn run_large_dataset_boundary() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + // Large stored portion + for i in 0..10000 { + vec.push(i); + } + vec.write()?; + + // Small pushed portion + for i in 10000..10100 { + vec.push(i); + } + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 10100); + + for (i, &val) in collected.iter().enumerate() { + assert_eq!(val, i as i32); + } + Ok(()) + } + + fn run_skip_take_complex() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..8000 { + vec.push(i); + } + vec.write()?; + + for i in 8000..12000 { + vec.push(i); + } + + // skip(7000).take(3000).skip(500).take(1000) => collect_range(7500, 8500) + let collected = vec.collect_range(7500, 8500); + + assert_eq!(collected.len(), 1000); + assert_eq!(collected[0], 7500); + assert_eq!(collected[999], 8499); + Ok(()) + } + + // ============================================================================ + // Test instantiation for each vec type + // ============================================================================ + + mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } + + #[cfg(feature = "zerocopy")] + mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } + + #[cfg(feature = "pco")] + mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } + + #[cfg(feature = "lz4")] + mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } + + #[cfg(feature = "zstd")] + mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } + + // ============================================================================ + // EagerVec Tests (wrapping different underlying vec types) + // ============================================================================ + + #[cfg(feature = "zerocopy")] + mod eager_zerocopy { + use super::*; + use vecdb::{EagerVec, ZeroCopyVec}; + type V = EagerVec>; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } + + #[cfg(feature = "pco")] + mod eager_pco { + use super::*; + use vecdb::{EagerVec, PcoVec}; + type V = EagerVec>; + + #[test] + fn only_stored() -> Result<()> { + run_only_stored::() + } + #[test] + fn only_pushed() -> Result<()> { + run_only_pushed::() + } + #[test] + fn stored_and_pushed() -> Result<()> { + run_stored_and_pushed::() + } + #[test] + fn skip_across_boundary() -> Result<()> { + run_skip_across_boundary::() + } + #[test] + fn take_across_boundary() -> Result<()> { + run_take_across_boundary::() + } + #[test] + fn nth_across_boundary() -> Result<()> { + run_nth_across_boundary::() + } + #[test] + fn set_position_to_pushed() -> Result<()> { + run_set_position_to_pushed::() + } + #[test] + fn last_in_pushed() -> Result<()> { + run_last_in_pushed::() + } + #[test] + fn last_in_stored() -> Result<()> { + run_last_in_stored::() + } + #[test] + fn exact_size_with_pushed() -> Result<()> { + run_exact_size_with_pushed::() + } + #[test] + fn large_dataset_boundary() -> Result<()> { + run_large_dataset_boundary::() + } + #[test] + fn skip_take_complex() -> Result<()> { + run_skip_take_complex::() + } + } +} + +// ============================================================================ +// Raw-specific Tests (holes and updates - only BytesVec and ZeroCopyVec) +// ============================================================================ + +mod raw_features { + use super::*; + use std::ops::DerefMut; + use vecdb::{BytesVec, ReadWriteRawVec}; + + // Generic test functions for raw vecs + + fn run_iter_skips_holes() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10 { + vec.push(i); + } + vec.write()?; + + // Delete some values (create holes) + vec.deref_mut().delete_at(3); + vec.deref_mut().delete_at(5); + vec.deref_mut().delete_at(7); + + let collected: Vec = vec.collect(); + // Should skip holes: 0,1,2,4,6,8,9 + assert_eq!(collected, vec![0, 1, 2, 4, 6, 8, 9]); + Ok(()) + } + + fn run_iter_with_updates() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10 { + vec.push(i); + } + vec.write()?; + + // Update some values + vec.deref_mut().update_at(2, 200)?; + vec.deref_mut().update_at(5, 500)?; + vec.deref_mut().update_at(8, 800)?; + + let collected: Vec = vec.collect(); + assert_eq!(collected, vec![0, 1, 200, 3, 4, 500, 6, 7, 800, 9]); + Ok(()) + } + + fn run_iter_with_holes_and_updates() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10 { + vec.push(i); + } + vec.write()?; + + // Create holes and updates + vec.deref_mut().delete_at(1); + vec.deref_mut().delete_at(3); + vec.deref_mut().update_at(2, 200)?; + vec.deref_mut().update_at(5, 500)?; + + let collected: Vec = vec.collect(); + // Should be: 0, (skip 1), 200, (skip 3), 4, 500, 6, 7, 8, 9 + assert_eq!(collected, vec![0, 200, 4, 500, 6, 7, 8, 9]); + Ok(()) + } + + fn run_iter_holes_and_pushed() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..5 { + vec.push(i); + } + vec.write()?; + + // Create holes in stored data + vec.deref_mut().delete_at(1); + vec.deref_mut().delete_at(3); + + // Push more data + for i in 5..10 { + vec.push(i); + } + + let collected: Vec = vec.collect(); + // Should be: 0, (skip 1), 2, (skip 3), 4, 5, 6, 7, 8, 9 + assert_eq!(collected, vec![0, 2, 4, 5, 6, 7, 8, 9]); + Ok(()) + } + + fn run_iter_updates_and_pushed() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..5 { + vec.push(i); + } + vec.write()?; + + // Update some stored values + vec.deref_mut().update_at(1, 100)?; + vec.deref_mut().update_at(3, 300)?; + + // Push more data + for i in 5..10 { + vec.push(i); + } + + let collected: Vec = vec.collect(); + assert_eq!(collected, vec![0, 100, 2, 300, 4, 5, 6, 7, 8, 9]); + Ok(()) + } + + fn run_iter_skip_over_holes() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..20 { + vec.push(i); + } + vec.write()?; + + // Create holes at indices 5, 6, 7 + vec.deref_mut().delete_at(5); + vec.deref_mut().delete_at(6); + vec.deref_mut().delete_at(7); + + // Skip past the holes — collect skips holes automatically + let collected: Vec = vec.collect(); + // Should be: 0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 + // (holes at 5,6,7 are skipped) + assert_eq!(collected[5..10], [8, 9, 10, 11, 12]); + Ok(()) + } + + fn run_fill_holes() -> Result<()> + where + V: StoredVec + DerefMut, + V::Target: RawVecOps, + { + let (db, _temp) = setup_db()?; + let mut vec = V::forced_import(&db, "test", Version::ONE)?; + + for i in 0..10 { + vec.push(i); + } + vec.write()?; + + // Create holes + vec.deref_mut().delete_at(2); + vec.deref_mut().delete_at(5); + + // Fill first hole + let idx = vec.deref_mut().fill_first_hole_or_push(999)?; + assert_eq!(idx, 2); + + let collected: Vec = vec.collect(); + // 0,1,999,3,4,(skip 5),6,7,8,9 + assert_eq!(collected, vec![0, 1, 999, 3, 4, 6, 7, 8, 9]); + Ok(()) + } + + // Helper trait for raw vec operations + pub trait RawVecOps { + fn delete_at(&mut self, index: usize); + fn update_at(&mut self, index: usize, value: i32) -> Result<()>; + fn fill_first_hole_or_push(&mut self, value: i32) -> Result; + } + + impl RawVecOps for ReadWriteRawVec + where + I: vecdb::VecIndex, + T: vecdb::VecValue + From + Into, + S: vecdb::RawStrategy, + { + fn delete_at(&mut self, index: usize) { + vecdb::ReadWriteRawVec::delete_at(self, index) + } + fn update_at(&mut self, index: usize, value: i32) -> Result<()> { + vecdb::ReadWriteRawVec::update_at(self, index, T::from(value)) + } + fn fill_first_hole_or_push(&mut self, value: i32) -> Result { + vecdb::ReadWriteRawVec::fill_first_hole_or_push(self, T::from(value)) + .map(|i| i.to_usize()) + } + } + + // ============================================================================ + // BytesVec Tests + // ============================================================================ + + mod bytes { + use super::*; + type V = BytesVec; + + #[test] + fn iter_skips_holes() -> Result<()> { + run_iter_skips_holes::() + } + #[test] + fn iter_with_updates() -> Result<()> { + run_iter_with_updates::() + } + #[test] + fn iter_with_holes_and_updates() -> Result<()> { + run_iter_with_holes_and_updates::() + } + #[test] + fn iter_holes_and_pushed() -> Result<()> { + run_iter_holes_and_pushed::() + } + #[test] + fn iter_updates_and_pushed() -> Result<()> { + run_iter_updates_and_pushed::() + } + #[test] + fn iter_skip_over_holes() -> Result<()> { + run_iter_skip_over_holes::() + } + #[test] + fn fill_holes() -> Result<()> { + run_fill_holes::() + } + } + + // ============================================================================ + // ZeroCopyVec Tests + // ============================================================================ + + #[cfg(feature = "zerocopy")] + mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn iter_skips_holes() -> Result<()> { + run_iter_skips_holes::() + } + #[test] + fn iter_with_updates() -> Result<()> { + run_iter_with_updates::() + } + #[test] + fn iter_with_holes_and_updates() -> Result<()> { + run_iter_with_holes_and_updates::() + } + #[test] + fn iter_holes_and_pushed() -> Result<()> { + run_iter_holes_and_pushed::() + } + #[test] + fn iter_updates_and_pushed() -> Result<()> { + run_iter_updates_and_pushed::() + } + #[test] + fn iter_skip_over_holes() -> Result<()> { + run_iter_skip_over_holes::() + } + #[test] + fn fill_holes() -> Result<()> { + run_fill_holes::() + } + } +} diff --git a/crates/vecdb/tests/non_copy.rs b/crates/vecdb/tests/non_copy.rs new file mode 100644 index 000000000..892688f01 --- /dev/null +++ b/crates/vecdb/tests/non_copy.rs @@ -0,0 +1,42 @@ +#![cfg(feature = "lz4")] + +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, Bytes, Database, ImportableVec, LZ4Vec, ReadableVec, Version, WritableVec, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct HeapValue(Box); + +impl Bytes for HeapValue { + type Array = [u8; 8]; + + fn to_bytes(&self) -> Self::Array { + self.0.to_le_bytes() + } + + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + let bytes: [u8; 8] = bytes.try_into().map_err(|_| vecdb::Error::WrongLength { + expected: 8, + received: bytes.len(), + })?; + Ok(Self(Box::new(u64::from_le_bytes(bytes)))) + } +} + +#[test] +fn compressed_fold_clones_non_copy_values() -> vecdb::Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + let mut vec: LZ4Vec = LZ4Vec::import(&db, "heap", Version::ONE)?; + + for value in 0..5_000 { + vec.push(HeapValue(Box::new(value))); + } + vec.write()?; + + let sum = vec.fold(0_u64, |sum, value| sum + *value.0); + assert_eq!(sum, (0..5_000_u64).sum::()); + + Ok(()) +} diff --git a/crates/vecdb/tests/raw_last_page.rs b/crates/vecdb/tests/raw_last_page.rs new file mode 100644 index 000000000..4747d0867 --- /dev/null +++ b/crates/vecdb/tests/raw_last_page.rs @@ -0,0 +1,1374 @@ +//! Tests for the "raw last page" optimization in compressed vectors. +//! +//! Compressed vectors store full pages compressed but keep the last partial page +//! as raw (uncompressed) bytes. This avoids recompression on every append. +//! +//! A fast-append path exists: when the last page is raw, not truncated, and the +//! new data fits within the page, bytes are appended directly without reading +//! back the existing page data. +//! +//! These tests cover: +//! - Small writes (all raw, never filling a page) +//! - Fast-append path (multiple small writes to the same raw page) +//! - Full page compression + raw tail +//! - Exact page boundary (no raw tail) +//! - Fast-append overflow (raw page fills up, triggers compression) +//! - Incremental growth across many pages +//! - Truncation into raw / compressed pages / exact boundaries +//! - Reset clearing raw pages +//! - Reads spanning compressed and raw pages +//! - Write-reopen-append cycles (simulates real usage) +//! - No-op writes on raw pages +//! - fold/iteration over mixed page types + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ReadableVec, Result, StoredVec, Version}; + +const PER_PAGE_U32: usize = 16 * 1024 / size_of::(); // 4096 + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +// ============================================================================ +// Small writes stay raw, data survives reopen +// ============================================================================ + +fn test_small_write_raw_survives_reopen() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let values: Vec = (0..100).collect(); + + { + let mut vec: V = V::forced_import_with(options)?; + for &v in &values { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), 100); + assert_eq!(vec.collect(), values); + } + + // Reopen and verify + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.stored_len(), 100); + assert_eq!(vec.collect(), values); + assert_eq!(vec.collect_range(0, 1), vec![0]); + assert_eq!(vec.collect_range(50, 53), vec![50, 51, 52]); + assert_eq!(vec.collect_range(99, 100), vec![99]); + } + + Ok(()) +} + +// ============================================================================ +// Fast-append path: multiple small writes to same raw page +// ============================================================================ + +fn test_fast_append_multiple_small_writes() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + // Write 10 values, flush → raw page + for v in 0..10u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), 10); + + // Write 10 more → fast append (raw page, room left) + for v in 10..20u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), 20); + assert_eq!(vec.collect(), (0..20).collect::>()); + + // Write 10 more again → fast append + for v in 20..30u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), 30); + assert_eq!(vec.collect(), (0..30).collect::>()); + + // Range reads across appended data + assert_eq!(vec.collect_range(5, 15), (5..15).collect::>()); + assert_eq!(vec.collect_range(15, 25), (15..25).collect::>()); + + Ok(()) +} + +// ============================================================================ +// Fast-append survives reopen and continued appending +// ============================================================================ + +fn test_fast_append_survives_reopen() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + for v in 0..50u32 { + vec.push(v); + } + vec.write()?; + + // Fast append + for v in 50..100u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.collect(), (0..100).collect::>()); + } + + // Reopen and verify the raw page was persisted correctly + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.stored_len(), 100); + assert_eq!(vec.collect(), (0..100).collect::>()); + } + + // Reopen and continue appending (should fast-append to existing raw page) + { + let mut vec: V = V::forced_import_with(options)?; + for v in 100..150u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.collect(), (0..150).collect::>()); + } + + // Final reopen + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.collect(), (0..150).collect::>()); + } + + Ok(()) +} + +// ============================================================================ +// Full page gets compressed, partial tail stays raw +// ============================================================================ + +fn test_full_page_compressed_partial_raw() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let count = PER_PAGE_U32 + 100; + let values: Vec = (0..count as u32).collect(); + + { + let mut vec: V = V::forced_import_with(options)?; + for &v in &values { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), count); + assert_eq!(vec.collect(), values); + + // Read across compressed→raw page boundary + let b = PER_PAGE_U32; + assert_eq!( + vec.collect_range(b - 2, b + 2), + vec![(b - 2) as u32, (b - 1) as u32, b as u32, (b + 1) as u32] + ); + } + + // Reopen and verify both compressed and raw pages survive + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.stored_len(), count); + assert_eq!(vec.collect(), values); + } + + Ok(()) +} + +// ============================================================================ +// Exact page boundary — no raw tail +// ============================================================================ + +fn test_exact_page_boundary() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let count = PER_PAGE_U32; + let values: Vec = (0..count as u32).collect(); + + { + let mut vec: V = V::forced_import_with(options)?; + for &v in &values { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), count); + assert_eq!(vec.collect(), values); + } + + // Reopen + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.stored_len(), count); + assert_eq!(vec.collect(), values); + } + + // Append a few more → creates a new raw page + { + let mut vec: V = V::forced_import_with(options)?; + for v in count as u32..(count + 5) as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.collect(), (0..(count + 5) as u32).collect::>()); + } + + // Reopen and verify + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.collect(), (0..(count + 5) as u32).collect::>()); + } + + Ok(()) +} + +// ============================================================================ +// Fast-append overflow: fills the raw page, triggers compression +// ============================================================================ + +fn test_fast_append_overflow() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + // Write almost a full page + let initial = PER_PAGE_U32 - 10; + for v in 0..initial as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), initial); + + // Push 20 more → exceeds page capacity, can't fast-append + // Should go through normal path: compress full page, raw tail of 10 + for v in initial as u32..(initial + 20) as u32 { + vec.push(v); + } + vec.write()?; + + let total = initial + 20; + let expected: Vec = (0..total as u32).collect(); + assert_eq!(vec.stored_len(), total); + assert_eq!(vec.collect(), expected); + + // Verify reads across compressed/raw boundary + let b = PER_PAGE_U32; + assert_eq!( + vec.collect_range(b - 2, b + 2), + vec![(b - 2) as u32, (b - 1) as u32, b as u32, (b + 1) as u32] + ); + + Ok(()) +} + +// ============================================================================ +// Fast-append fills exactly to page boundary +// ============================================================================ + +fn test_fast_append_fills_exactly() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + // Write almost a full page + let initial = PER_PAGE_U32 - 10; + for v in 0..initial as u32 { + vec.push(v); + } + vec.write()?; + + // Push exactly 10 more → fills page exactly + // fast-append condition is `partial_len + pushed_len < PER_PAGE` (strict <), + // so this goes through the normal path and compresses the full page + for v in initial as u32..PER_PAGE_U32 as u32 { + vec.push(v); + } + vec.write()?; + + assert_eq!(vec.stored_len(), PER_PAGE_U32); + assert_eq!( + vec.collect(), + (0..PER_PAGE_U32 as u32).collect::>() + ); + + // Append more → new raw page + for v in PER_PAGE_U32 as u32..(PER_PAGE_U32 + 5) as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!( + vec.collect(), + (0..(PER_PAGE_U32 + 5) as u32).collect::>() + ); + + Ok(()) +} + +// ============================================================================ +// Incremental growth across multiple pages +// ============================================================================ + +fn test_incremental_growth_across_pages() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + let chunk_size = 1000; + let total_target = PER_PAGE_U32 * 2 + 500; + let num_chunks = total_target.div_ceil(chunk_size); + let total = num_chunks * chunk_size; + + for chunk_i in 0..num_chunks { + let start = chunk_i * chunk_size; + for v in start..(start + chunk_size) { + vec.push(v as u32); + } + vec.write()?; + + let expected: Vec = (0..(start + chunk_size) as u32).collect(); + assert_eq!(vec.collect(), expected, "Mismatch after chunk {}", chunk_i); + } + + // Final full verification + assert_eq!(vec.collect(), (0..total as u32).collect::>()); + + // Reads at page boundaries + assert_eq!(vec.collect_range(0, 10), (0..10).collect::>()); + assert_eq!( + vec.collect_range(PER_PAGE_U32 - 5, PER_PAGE_U32 + 5), + ((PER_PAGE_U32 - 5) as u32..(PER_PAGE_U32 + 5) as u32).collect::>() + ); + assert_eq!( + vec.collect_range(PER_PAGE_U32 * 2 - 5, PER_PAGE_U32 * 2 + 5), + ((PER_PAGE_U32 * 2 - 5) as u32..(PER_PAGE_U32 * 2 + 5) as u32).collect::>() + ); + + Ok(()) +} + +// ============================================================================ +// Truncation into a raw last page +// ============================================================================ + +fn test_truncate_into_raw_page() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + let count = PER_PAGE_U32 + 500; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + // Truncate to within the raw tail page + let truncate_to = PER_PAGE_U32 + 200; + vec.truncate_if_needed(truncate_to)?; + vec.write()?; + assert_eq!(vec.collect(), (0..truncate_to as u32).collect::>()); + } + + // Reopen and verify + { + let vec: V = V::forced_import_with(options)?; + let truncate_to = PER_PAGE_U32 + 200; + assert_eq!(vec.stored_len(), truncate_to); + assert_eq!(vec.collect(), (0..truncate_to as u32).collect::>()); + } + + // Append after truncation into raw page + { + let mut vec: V = V::forced_import_with(options)?; + let truncate_to = PER_PAGE_U32 + 200; + for v in truncate_to as u32..(truncate_to + 50) as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!( + vec.collect(), + (0..(truncate_to + 50) as u32).collect::>() + ); + } + + Ok(()) +} + +// ============================================================================ +// Truncation to exact page boundary +// ============================================================================ + +fn test_truncate_to_page_boundary() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + let count = PER_PAGE_U32 + 500; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + // Truncate to the first page boundary (removes raw tail entirely) + vec.truncate_if_needed(PER_PAGE_U32)?; + vec.write()?; + assert_eq!( + vec.collect(), + (0..PER_PAGE_U32 as u32).collect::>() + ); + } + + // Reopen + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.stored_len(), PER_PAGE_U32); + assert_eq!( + vec.collect(), + (0..PER_PAGE_U32 as u32).collect::>() + ); + } + + // Append after truncation to boundary → new raw page + { + let mut vec: V = V::forced_import_with(options)?; + for v in PER_PAGE_U32 as u32..(PER_PAGE_U32 + 10) as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!( + vec.collect(), + (0..(PER_PAGE_U32 + 10) as u32).collect::>() + ); + } + + Ok(()) +} + +// ============================================================================ +// Truncation into a compressed page (removes raw tail and part of compressed) +// ============================================================================ + +fn test_truncate_into_compressed_page() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + let count = PER_PAGE_U32 + 500; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + // Truncate to half the first (compressed) page + let truncate_to = PER_PAGE_U32 / 2; + vec.truncate_if_needed(truncate_to)?; + vec.write()?; + assert_eq!(vec.collect(), (0..truncate_to as u32).collect::>()); + } + + // Reopen + { + let vec: V = V::forced_import_with(options)?; + let truncate_to = PER_PAGE_U32 / 2; + assert_eq!(vec.stored_len(), truncate_to); + assert_eq!(vec.collect(), (0..truncate_to as u32).collect::>()); + } + + // Append after truncation into compressed page + { + let mut vec: V = V::forced_import_with(options)?; + let truncate_to = PER_PAGE_U32 / 2; + for v in truncate_to as u32..(truncate_to + 100) as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!( + vec.collect(), + (0..(truncate_to + 100) as u32).collect::>() + ); + } + + Ok(()) +} + +// ============================================================================ +// Reset clears raw pages +// ============================================================================ + +fn test_reset_clears_raw_pages() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + for v in 0..100u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), 100); + + vec.reset()?; + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.len(), 0); + assert!(vec.collect().is_empty()); + + // Write new data after reset + for v in 1000..1050u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.collect(), (1000..1050).collect::>()); + + Ok(()) +} + +// ============================================================================ +// Reset after multi-page data (compressed + raw) +// ============================================================================ + +fn test_reset_after_multi_page() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + let count = PER_PAGE_U32 + 200; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + vec.reset()?; + assert_eq!(vec.len(), 0); + assert!(vec.collect().is_empty()); + + // Write again + for v in 0..50u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.collect(), (0..50).collect::>()); + } + + // Reopen + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.collect(), (0..50).collect::>()); + } + + Ok(()) +} + +// ============================================================================ +// Read spanning compressed and raw pages +// ============================================================================ + +fn test_read_spanning_compressed_and_raw() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + let count = PER_PAGE_U32 + 200; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + // Spanning compressed→raw boundary + let from = PER_PAGE_U32 - 50; + let to = PER_PAGE_U32 + 50; + assert_eq!( + vec.collect_range(from, to), + (from as u32..to as u32).collect::>() + ); + + // Entirely within compressed page + assert_eq!(vec.collect_range(0, 100), (0..100).collect::>()); + + // Entirely within raw page + assert_eq!( + vec.collect_range(PER_PAGE_U32, PER_PAGE_U32 + 100), + (PER_PAGE_U32 as u32..(PER_PAGE_U32 + 100) as u32).collect::>() + ); + + // Full read + assert_eq!(vec.collect(), (0..count as u32).collect::>()); + + Ok(()) +} + +// ============================================================================ +// Multiple pages with raw tail +// ============================================================================ + +fn test_multiple_pages_with_raw_tail() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + // 3 full pages + partial raw tail + let count = PER_PAGE_U32 * 3 + 777; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.stored_len(), count); + + // Read from each page + for page in 0..4 { + let start = page * PER_PAGE_U32; + let end = (start + 10).min(count); + let result = vec.collect_range(start, end); + let expected: Vec = (start as u32..end as u32).collect(); + assert_eq!(result, expected, "Page {} read mismatch", page); + } + + // Read spanning each page boundary + for boundary_page in 1..=3 { + let mid = boundary_page * PER_PAGE_U32; + let from = mid - 5; + let to = (mid + 5).min(count); + assert_eq!( + vec.collect_range(from, to), + (from as u32..to as u32).collect::>(), + "Boundary {} read mismatch", + boundary_page + ); + } + + // Full read + assert_eq!(vec.collect(), (0..count as u32).collect::>()); + + Ok(()) +} + +// ============================================================================ +// Write-reopen-append cycle (simulates real usage patterns) +// ============================================================================ + +fn test_write_reopen_append_cycle() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + let mut total = 0u32; + for cycle in 0..20u32 { + let mut vec: V = V::forced_import_with(options)?; + let count = 100 + cycle * 50; + for _ in 0..count { + vec.push(total); + total += 1; + } + vec.write()?; + assert_eq!( + vec.collect(), + (0..total).collect::>(), + "Cycle {}", + cycle + ); + } + + // Final verification after all cycles + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.stored_len(), total as usize); + assert_eq!(vec.collect(), (0..total).collect::>()); + } + + Ok(()) +} + +// ============================================================================ +// Write-reopen-append cycle crossing page boundaries +// ============================================================================ + +fn test_write_reopen_cycle_crossing_pages() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + // Use a chunk size that doesn't align with PER_PAGE to ensure we cross boundaries + let chunk = PER_PAGE_U32 / 3 + 7; + let mut total = 0u32; + for cycle in 0..10u32 { + let mut vec: V = V::forced_import_with(options)?; + for _ in 0..chunk { + vec.push(total); + total += 1; + } + vec.write()?; + assert_eq!( + vec.collect(), + (0..total).collect::>(), + "Cycle {} (total={})", + cycle, + total + ); + } + + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.collect(), (0..total).collect::>()); + } + + Ok(()) +} + +// ============================================================================ +// No-op write on a raw page +// ============================================================================ + +fn test_noop_write_on_raw_page() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + for v in 0..50u32 { + vec.push(v); + } + vec.write()?; + + // Write again with no changes + let changed = vec.write()?; + assert!(!changed); + assert_eq!(vec.collect(), (0..50).collect::>()); + + Ok(()) +} + +// ============================================================================ +// No-op write after multi-page data +// ============================================================================ + +fn test_noop_write_after_multi_page() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + let count = PER_PAGE_U32 + 100; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + let changed = vec.write()?; + assert!(!changed); + assert_eq!(vec.collect(), (0..count as u32).collect::>()); + + Ok(()) +} + +// ============================================================================ +// fold/iteration over mixed compressed+raw pages +// ============================================================================ + +fn test_fold_over_mixed_pages() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + let count = PER_PAGE_U32 + 500; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + // Sum all values using for_each + let expected_sum: u64 = (0..count as u64).sum(); + let mut actual_sum = 0u64; + vec.for_each(|v: u32| actual_sum += v as u64); + assert_eq!(actual_sum, expected_sum); + + // fold_range spanning the compressed→raw boundary + let from = PER_PAGE_U32 - 100; + let to = PER_PAGE_U32 + 100; + let range_sum = vec.fold_range_at(from, to, 0u64, |acc, v: u32| acc + v as u64); + let expected_range_sum: u64 = (from as u64..to as u64).sum(); + assert_eq!(range_sum, expected_range_sum); + + Ok(()) +} + +// ============================================================================ +// Pushed (un-flushed) values mixed with stored raw page +// ============================================================================ + +fn test_pushed_and_stored_raw_page() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + // Write some values → stored as raw page + for v in 0..100u32 { + vec.push(v); + } + vec.write()?; + + // Push more without writing → pushed buffer + for v in 100..150u32 { + vec.push(v); + } + + assert_eq!(vec.stored_len(), 100); + assert_eq!(vec.pushed_len(), 50); + assert_eq!(vec.len(), 150); + + // Read spanning stored raw + pushed + assert_eq!(vec.collect_range(90, 110), (90..110).collect::>()); + assert_eq!(vec.collect(), (0..150).collect::>()); + + // Now write + vec.write()?; + assert_eq!(vec.stored_len(), 150); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.collect(), (0..150).collect::>()); + + Ok(()) +} + +// ============================================================================ +// Pushed values mixed with stored compressed + raw pages +// ============================================================================ + +fn test_pushed_and_stored_mixed_pages() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + let count = PER_PAGE_U32 + 100; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + // Push more without writing + for v in count as u32..(count + 50) as u32 { + vec.push(v); + } + + assert_eq!(vec.stored_len(), count); + assert_eq!(vec.pushed_len(), 50); + assert_eq!(vec.len(), count + 50); + + // Read spanning raw page → pushed buffer + assert_eq!( + vec.collect_range(count - 10, count + 10), + ((count - 10) as u32..(count + 10) as u32).collect::>() + ); + + // Read spanning compressed → raw → pushed + let from = PER_PAGE_U32 - 5; + let to = count + 5; + assert_eq!( + vec.collect_range(from, to), + (from as u32..to as u32).collect::>() + ); + + Ok(()) +} + +// ============================================================================ +// Single value writes (extreme fast-append case) +// ============================================================================ + +fn test_single_value_writes() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + // Push and write one value at a time, 50 times + for v in 0..50u32 { + vec.push(v); + vec.write()?; + } + + assert_eq!(vec.stored_len(), 50); + assert_eq!(vec.collect(), (0..50).collect::>()); + + // Reopen + drop(vec); + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.collect(), (0..50).collect::>()); + + Ok(()) +} + +// ============================================================================ +// Truncate all then rebuild (edge case) +// ============================================================================ + +fn test_truncate_to_zero_then_rebuild() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + + { + let mut vec: V = V::forced_import_with(options)?; + let count = PER_PAGE_U32 + 100; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + vec.truncate_if_needed(0)?; + assert_eq!(vec.len(), 0); + + // Rebuild with different data + for v in 5000..5100u32 { + vec.push(v); + } + vec.write()?; + assert_eq!(vec.collect(), (5000..5100).collect::>()); + } + + // Reopen + { + let vec: V = V::forced_import_with(options)?; + assert_eq!(vec.collect(), (5000..5100).collect::>()); + } + + Ok(()) +} + +// ============================================================================ +// Read-only clone reads mixed pages correctly +// ============================================================================ + +fn test_read_only_clone_mixed_pages() -> Result<()> +where + V: StoredVec, +{ + let (db, _tmp) = setup_db()?; + let options = (&db, "vec", Version::TWO).into(); + let mut vec: V = V::forced_import_with(options)?; + + let count = PER_PAGE_U32 + 200; + for v in 0..count as u32 { + vec.push(v); + } + vec.write()?; + + let ro = vec.read_only_clone(); + assert_eq!(ro.collect(), (0..count as u32).collect::>()); + + // Read across boundary via read-only clone + let b = PER_PAGE_U32; + assert_eq!( + ro.collect_range(b - 5, b + 5), + ((b - 5) as u32..(b + 5) as u32).collect::>() + ); + + Ok(()) +} + +// ============================================================================ +// Test instantiation for each compression strategy +// ============================================================================ + +#[cfg(feature = "pco")] +mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn small_write_raw_survives_reopen() -> Result<()> { + test_small_write_raw_survives_reopen::() + } + #[test] + fn fast_append_multiple_small_writes() -> Result<()> { + test_fast_append_multiple_small_writes::() + } + #[test] + fn fast_append_survives_reopen() -> Result<()> { + test_fast_append_survives_reopen::() + } + #[test] + fn full_page_compressed_partial_raw() -> Result<()> { + test_full_page_compressed_partial_raw::() + } + #[test] + fn exact_page_boundary() -> Result<()> { + test_exact_page_boundary::() + } + #[test] + fn fast_append_overflow() -> Result<()> { + test_fast_append_overflow::() + } + #[test] + fn fast_append_fills_exactly() -> Result<()> { + test_fast_append_fills_exactly::() + } + #[test] + fn incremental_growth_across_pages() -> Result<()> { + test_incremental_growth_across_pages::() + } + #[test] + fn truncate_into_raw_page() -> Result<()> { + test_truncate_into_raw_page::() + } + #[test] + fn truncate_to_page_boundary() -> Result<()> { + test_truncate_to_page_boundary::() + } + #[test] + fn truncate_into_compressed_page() -> Result<()> { + test_truncate_into_compressed_page::() + } + #[test] + fn reset_clears_raw_pages() -> Result<()> { + test_reset_clears_raw_pages::() + } + #[test] + fn reset_after_multi_page() -> Result<()> { + test_reset_after_multi_page::() + } + #[test] + fn read_spanning_compressed_and_raw() -> Result<()> { + test_read_spanning_compressed_and_raw::() + } + #[test] + fn multiple_pages_with_raw_tail() -> Result<()> { + test_multiple_pages_with_raw_tail::() + } + #[test] + fn write_reopen_append_cycle() -> Result<()> { + test_write_reopen_append_cycle::() + } + #[test] + fn write_reopen_cycle_crossing_pages() -> Result<()> { + test_write_reopen_cycle_crossing_pages::() + } + #[test] + fn noop_write_on_raw_page() -> Result<()> { + test_noop_write_on_raw_page::() + } + #[test] + fn noop_write_after_multi_page() -> Result<()> { + test_noop_write_after_multi_page::() + } + #[test] + fn fold_over_mixed_pages() -> Result<()> { + test_fold_over_mixed_pages::() + } + #[test] + fn pushed_and_stored_raw_page() -> Result<()> { + test_pushed_and_stored_raw_page::() + } + #[test] + fn pushed_and_stored_mixed_pages() -> Result<()> { + test_pushed_and_stored_mixed_pages::() + } + #[test] + fn single_value_writes() -> Result<()> { + test_single_value_writes::() + } + #[test] + fn truncate_to_zero_then_rebuild() -> Result<()> { + test_truncate_to_zero_then_rebuild::() + } + #[test] + fn read_only_clone_mixed_pages() -> Result<()> { + test_read_only_clone_mixed_pages::() + } +} + +#[cfg(feature = "lz4")] +mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn small_write_raw_survives_reopen() -> Result<()> { + test_small_write_raw_survives_reopen::() + } + #[test] + fn fast_append_multiple_small_writes() -> Result<()> { + test_fast_append_multiple_small_writes::() + } + #[test] + fn fast_append_survives_reopen() -> Result<()> { + test_fast_append_survives_reopen::() + } + #[test] + fn full_page_compressed_partial_raw() -> Result<()> { + test_full_page_compressed_partial_raw::() + } + #[test] + fn exact_page_boundary() -> Result<()> { + test_exact_page_boundary::() + } + #[test] + fn fast_append_overflow() -> Result<()> { + test_fast_append_overflow::() + } + #[test] + fn fast_append_fills_exactly() -> Result<()> { + test_fast_append_fills_exactly::() + } + #[test] + fn incremental_growth_across_pages() -> Result<()> { + test_incremental_growth_across_pages::() + } + #[test] + fn truncate_into_raw_page() -> Result<()> { + test_truncate_into_raw_page::() + } + #[test] + fn truncate_to_page_boundary() -> Result<()> { + test_truncate_to_page_boundary::() + } + #[test] + fn truncate_into_compressed_page() -> Result<()> { + test_truncate_into_compressed_page::() + } + #[test] + fn reset_clears_raw_pages() -> Result<()> { + test_reset_clears_raw_pages::() + } + #[test] + fn reset_after_multi_page() -> Result<()> { + test_reset_after_multi_page::() + } + #[test] + fn read_spanning_compressed_and_raw() -> Result<()> { + test_read_spanning_compressed_and_raw::() + } + #[test] + fn multiple_pages_with_raw_tail() -> Result<()> { + test_multiple_pages_with_raw_tail::() + } + #[test] + fn write_reopen_append_cycle() -> Result<()> { + test_write_reopen_append_cycle::() + } + #[test] + fn write_reopen_cycle_crossing_pages() -> Result<()> { + test_write_reopen_cycle_crossing_pages::() + } + #[test] + fn noop_write_on_raw_page() -> Result<()> { + test_noop_write_on_raw_page::() + } + #[test] + fn noop_write_after_multi_page() -> Result<()> { + test_noop_write_after_multi_page::() + } + #[test] + fn fold_over_mixed_pages() -> Result<()> { + test_fold_over_mixed_pages::() + } + #[test] + fn pushed_and_stored_raw_page() -> Result<()> { + test_pushed_and_stored_raw_page::() + } + #[test] + fn pushed_and_stored_mixed_pages() -> Result<()> { + test_pushed_and_stored_mixed_pages::() + } + #[test] + fn single_value_writes() -> Result<()> { + test_single_value_writes::() + } + #[test] + fn truncate_to_zero_then_rebuild() -> Result<()> { + test_truncate_to_zero_then_rebuild::() + } + #[test] + fn read_only_clone_mixed_pages() -> Result<()> { + test_read_only_clone_mixed_pages::() + } +} + +#[cfg(feature = "zstd")] +mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn small_write_raw_survives_reopen() -> Result<()> { + test_small_write_raw_survives_reopen::() + } + #[test] + fn fast_append_multiple_small_writes() -> Result<()> { + test_fast_append_multiple_small_writes::() + } + #[test] + fn fast_append_survives_reopen() -> Result<()> { + test_fast_append_survives_reopen::() + } + #[test] + fn full_page_compressed_partial_raw() -> Result<()> { + test_full_page_compressed_partial_raw::() + } + #[test] + fn exact_page_boundary() -> Result<()> { + test_exact_page_boundary::() + } + #[test] + fn fast_append_overflow() -> Result<()> { + test_fast_append_overflow::() + } + #[test] + fn fast_append_fills_exactly() -> Result<()> { + test_fast_append_fills_exactly::() + } + #[test] + fn incremental_growth_across_pages() -> Result<()> { + test_incremental_growth_across_pages::() + } + #[test] + fn truncate_into_raw_page() -> Result<()> { + test_truncate_into_raw_page::() + } + #[test] + fn truncate_to_page_boundary() -> Result<()> { + test_truncate_to_page_boundary::() + } + #[test] + fn truncate_into_compressed_page() -> Result<()> { + test_truncate_into_compressed_page::() + } + #[test] + fn reset_clears_raw_pages() -> Result<()> { + test_reset_clears_raw_pages::() + } + #[test] + fn reset_after_multi_page() -> Result<()> { + test_reset_after_multi_page::() + } + #[test] + fn read_spanning_compressed_and_raw() -> Result<()> { + test_read_spanning_compressed_and_raw::() + } + #[test] + fn multiple_pages_with_raw_tail() -> Result<()> { + test_multiple_pages_with_raw_tail::() + } + #[test] + fn write_reopen_append_cycle() -> Result<()> { + test_write_reopen_append_cycle::() + } + #[test] + fn write_reopen_cycle_crossing_pages() -> Result<()> { + test_write_reopen_cycle_crossing_pages::() + } + #[test] + fn noop_write_on_raw_page() -> Result<()> { + test_noop_write_on_raw_page::() + } + #[test] + fn noop_write_after_multi_page() -> Result<()> { + test_noop_write_after_multi_page::() + } + #[test] + fn fold_over_mixed_pages() -> Result<()> { + test_fold_over_mixed_pages::() + } + #[test] + fn pushed_and_stored_raw_page() -> Result<()> { + test_pushed_and_stored_raw_page::() + } + #[test] + fn pushed_and_stored_mixed_pages() -> Result<()> { + test_pushed_and_stored_mixed_pages::() + } + #[test] + fn single_value_writes() -> Result<()> { + test_single_value_writes::() + } + #[test] + fn truncate_to_zero_then_rebuild() -> Result<()> { + test_truncate_to_zero_then_rebuild::() + } + #[test] + fn read_only_clone_mixed_pages() -> Result<()> { + test_read_only_clone_mixed_pages::() + } +} diff --git a/crates/vecdb/tests/raw_reader_cursor.rs b/crates/vecdb/tests/raw_reader_cursor.rs new file mode 100644 index 000000000..2bf555599 --- /dev/null +++ b/crates/vecdb/tests/raw_reader_cursor.rs @@ -0,0 +1,70 @@ +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{AnyStoredVec, BytesVec, ImportableVec, Result, Version, WritableVec}; + +#[test] +fn raw_reader_cursor_reads_persisted_values() -> Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + let mut vec = BytesVec::::import(&db, "values", Version::ONE)?; + + for value in 0..10 { + vec.push(value); + } + vec.write()?; + + // Reader cursors intentionally do not include later uncommitted values. + vec.push(10); + assert_eq!(vec.read_once(9)?, 9); + assert!(vec.read_once(10).is_err()); + let reader = vec.reader(); + assert_eq!(reader.try_get(10), None); + + assert_eq!(vec.get_append_only(9, &reader), Some(9)); + assert_eq!(vec.get_append_only(10, &reader), Some(10)); + assert_eq!(vec.get_append_only(11, &reader), None); + + let mut cursor = vec.reader().cursor(); + assert_eq!(cursor.position(), 0); + assert_eq!(cursor.remaining(), 10); + assert_eq!(cursor.get(7), Some(7)); + assert_eq!(cursor.position(), 0); + + cursor.advance(3); + assert_eq!(cursor.position(), 3); + assert_eq!(cursor.next(), Some(3)); + assert_eq!(cursor.position(), 4); + + assert_eq!( + cursor.fold(3, Vec::new(), |mut values, value| { + values.push(value); + values + }), + vec![4, 5, 6] + ); + assert_eq!(cursor.position(), 7); + + let mut tail = Vec::new(); + cursor.for_each(usize::MAX, |value| tail.push(value)); + assert_eq!(tail, vec![7, 8, 9]); + assert_eq!(cursor.position(), 10); + assert_eq!(cursor.remaining(), 0); + assert_eq!(cursor.next(), None); + + Ok(()) +} + +#[test] +#[cfg(debug_assertions)] +#[should_panic(expected = "get_append_only requires a vector without holes or updates")] +fn append_only_reader_rejects_dirty_vectors() { + let temp = TempDir::new().unwrap(); + let db = Database::open(temp.path()).unwrap(); + let mut vec = BytesVec::::import(&db, "values", Version::ONE).unwrap(); + vec.push(1); + vec.write().unwrap(); + + let reader = vec.reader(); + vec.update(0, 2).unwrap(); + let _ = vec.get_append_only(0, &reader); +} diff --git a/crates/vecdb/tests/rollback.rs b/crates/vecdb/tests/rollback.rs new file mode 100644 index 000000000..4e3859e6e --- /dev/null +++ b/crates/vecdb/tests/rollback.rs @@ -0,0 +1,1856 @@ +//! Generic rollback tests for all vec types. +//! +//! This module contains two sets of rollback tests: +//! 1. Generic rollback tests - work with ALL vec types (BytesVec, ZeroCopyVec, PcoVec, LZ4Vec, ZstdVec, EagerVec) +//! These use only push/truncate operations available on all vecs. +//! 2. Raw-only rollback tests - work with raw vecs (BytesVec, ZeroCopyVec) only +//! These test update/hole operations specific to raw vecs. + +use rawdb::Database; +use std::ops::DerefMut; +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, ImportOptions, ImportableVec, ReadableVec, Result, Stamp, StoredVec, VecReader, + Version, WritableVec, +}; + +// ============================================================================ +// Test Setup +// ============================================================================ + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +// ============================================================================ +// PART 1: Generic Rollback Tests (ALL vec types) +// ============================================================================ +// These tests use only push/truncate operations and work with any StoredVec. + +mod generic_rollback { + use super::*; + + fn import_with_changes(db: &Database, name: &str, changes: u16) -> Result + where + V: StoredVec, + { + let mut options: ImportOptions = (db, name, Version::TWO).into(); + options = options.with_saved_stamped_changes(changes); + V::forced_import_with(options) + } + + fn run_basic_rollback() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.stamp(), Stamp::new(1)); + + // Stamp 2: [0, 1, 2, 3, 4, 5, 6] + vec.push(5); + vec.push(6); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6]); + assert_eq!(vec.stamp(), Stamp::new(2)); + + // Rollback to stamp 1 + vec.rollback()?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_with_truncation() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [0, 1, 2, 3, 4, 5, 6, 7] + vec.push(5); + vec.push(6); + vec.push(7); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6, 7]); + + // Rollback - should restore to [0, 1, 2, 3, 4] + vec.rollback()?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_multiple_sequential_rollbacks() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [0, 1, 2, 3, 4, 5] + vec.push(5); + vec.stamped_write_with_changes(Stamp::new(2))?; + + // Stamp 3: [0, 1, 2, 3, 4, 5, 6] + vec.push(6); + vec.stamped_write_with_changes(Stamp::new(3))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6]); + + // Rollback to stamp 2 + vec.rollback()?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5]); + assert_eq!(vec.stamp(), Stamp::new(2)); + + // Rollback to stamp 1 + vec.rollback()?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_then_save_new_state() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [0, 1, 2, 3, 4, 5] + vec.push(5); + vec.stamped_write_with_changes(Stamp::new(2))?; + + // Rollback to stamp 1 + vec.rollback()?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + + // Save new stamp 2: [0, 1, 2, 3, 4, 99] + vec.push(99); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 99]); + assert_eq!(vec.stamp(), Stamp::new(2)); + + Ok(()) + } + + fn run_rollback_to_empty() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Stamp 1: [] + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.collect(), Vec::::new()); + + // Stamp 2: [0, 1, 2] + vec.push(0); + vec.push(1); + vec.push(2); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.collect(), vec![0, 1, 2]); + + // Rollback to empty + vec.rollback()?; + assert_eq!(vec.collect(), Vec::::new()); + + Ok(()) + } + + fn run_rollback_before() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Build stamps 1-5 + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + vec.push(5); + vec.stamped_write_with_changes(Stamp::new(2))?; + + vec.push(6); + vec.stamped_write_with_changes(Stamp::new(3))?; + + vec.push(7); + vec.stamped_write_with_changes(Stamp::new(4))?; + + vec.push(8); + vec.stamped_write_with_changes(Stamp::new(5))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8]); + + // Rollback before stamp 4 (should go to stamp 3) + let _ = vec.rollback_before(Stamp::new(4))?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6]); + assert_eq!(vec.stamp(), Stamp::new(3)); + + Ok(()) + } + + fn run_deep_rollback_chain() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Build chain of stamps with pushes only + vec.stamped_write_with_changes(Stamp::new(1))?; // [] + + vec.push(0); + vec.stamped_write_with_changes(Stamp::new(2))?; // [0] + + vec.push(1); + vec.stamped_write_with_changes(Stamp::new(3))?; // [0, 1] + + vec.push(2); + vec.stamped_write_with_changes(Stamp::new(4))?; // [0, 1, 2] + + vec.push(3); + vec.push(4); + vec.stamped_write_with_changes(Stamp::new(5))?; // [0, 1, 2, 3, 4] + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + + // Rollback through chain + vec.rollback()?; // -> 4 + assert_eq!(vec.collect(), vec![0, 1, 2]); + + vec.rollback()?; // -> 3 + assert_eq!(vec.collect(), vec![0, 1]); + + vec.rollback()?; // -> 2 + assert_eq!(vec.collect(), vec![0]); + + vec.rollback()?; // -> 1 + assert_eq!(vec.collect(), Vec::::new()); + + Ok(()) + } + + fn run_rollback_persistence() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + + // Create and populate + { + let mut vec = import_with_changes::(&db, "test", 10)?; + + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + vec.push(5); + vec.push(6); + vec.stamped_write_with_changes(Stamp::new(2))?; + + // Rollback and flush + vec.rollback()?; + vec.stamped_write_with_changes(Stamp::new(1))?; + } + + // Reopen and verify + { + let vec = import_with_changes::(&db, "test", 10)?; + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.stamp(), Stamp::new(1)); + } + + Ok(()) + } + + fn run_reset() -> Result<()> + where + V: StoredVec, + { + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "test", 10)?; + + // Add initial data and flush + for i in 0..10 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.len(), 10); + assert_eq!(vec.stored_len(), 10); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.collect(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + + // Add more data without flushing + vec.push(10); + vec.push(11); + assert_eq!(vec.len(), 12); + assert_eq!(vec.stored_len(), 10); + assert_eq!(vec.pushed_len(), 2); + + // Reset should clear everything + vec.reset()?; + assert_eq!(vec.len(), 0); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.collect(), Vec::::new()); + + // Should be able to add new data after reset + vec.push(100); + vec.push(101); + vec.push(102); + assert_eq!(vec.len(), 3); + assert_eq!(vec.stored_len(), 0); + assert_eq!(vec.pushed_len(), 3); + assert_eq!(vec.collect(), vec![100, 101, 102]); + + // Flush the new data + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.len(), 3); + assert_eq!(vec.stored_len(), 3); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.collect(), vec![100, 101, 102]); + + Ok(()) + } + + // Test modules for each vec type + mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } + + #[cfg(feature = "zerocopy")] + mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } + + #[cfg(feature = "pco")] + mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } + + #[cfg(feature = "lz4")] + mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } + + #[cfg(feature = "zstd")] + mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } + + #[cfg(feature = "zerocopy")] + mod eager_zerocopy { + use super::*; + use vecdb::{EagerVec, ZeroCopyVec}; + type V = EagerVec>; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } + + #[cfg(feature = "pco")] + mod eager_pco { + use super::*; + use vecdb::{EagerVec, PcoVec}; + type V = EagerVec>; + + #[test] + fn basic_rollback() -> Result<()> { + run_basic_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_persistence() -> Result<()> { + run_rollback_persistence::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + } +} + +// ============================================================================ +// PART 2: Raw-Only Rollback Tests (BytesVec and ZeroCopyVec) +// ============================================================================ +// These tests use update/hole operations specific to raw vecs. + +mod raw_rollback { + use super::*; + + // ============================================================================ + // Trait for raw vec rollback operations + // ============================================================================ + + /// Trait for raw vecs that support rollback operations. + pub trait RollbackVec: StoredVec + DerefMut + where + Self::Target: RollbackOps, + { + fn import_with_changes<'a>( + db: &'a Database, + name: &'a str, + changes: u16, + ) -> Result<(Self, ImportOptions<'a>)>; + } + + /// Operations required for rollback testing. + pub trait RollbackOps { + type Reader; + + fn update(&mut self, index: usize, value: u32) -> Result<()>; + fn take(&mut self, index: usize) -> Option; + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()>; + fn rollback(&mut self) -> Result<()>; + fn rollback_before(&mut self, stamp: Stamp) -> Result; + fn stamp(&self) -> Stamp; + fn stored_len(&self) -> usize; + fn collect(&self) -> Vec; + fn collect_holed(&self) -> Vec>; + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option; + fn reader(&self) -> Self::Reader; + } + + // ============================================================================ + // Implementations for ZeroCopyVec + // ============================================================================ + + #[cfg(feature = "zerocopy")] + use vecdb::{ReadWriteRawVec, ZeroCopyStrategy, ZeroCopyVec}; + + #[cfg(feature = "zerocopy")] + impl RollbackVec for ZeroCopyVec { + fn import_with_changes<'a>( + db: &'a Database, + name: &'a str, + changes: u16, + ) -> Result<(Self, ImportOptions<'a>)> { + let mut options: ImportOptions = (db, name, Version::TWO).into(); + options = options.with_saved_stamped_changes(changes); + let vec = Self::forced_import_with(options)?; + Ok((vec, options)) + } + } + + #[cfg(feature = "zerocopy")] + impl RollbackOps for ReadWriteRawVec> { + type Reader = VecReader>; + + fn update(&mut self, index: usize, value: u32) -> Result<()> { + ReadWriteRawVec::update(self, index, value) + } + + fn take(&mut self, index: usize) -> Option { + let reader = self.reader(); + let result = ReadWriteRawVec::take(self, index, &reader); + drop(reader); + result + } + + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + WritableVec::stamped_write_with_changes(self, stamp) + } + + fn rollback(&mut self) -> Result<()> { + WritableVec::rollback(self) + } + + fn rollback_before(&mut self, stamp: Stamp) -> Result { + WritableVec::rollback_before(self, stamp) + } + + fn stamp(&self) -> Stamp { + AnyStoredVec::stamp(self) + } + + fn stored_len(&self) -> usize { + AnyStoredVec::stored_len(self) + } + + fn collect(&self) -> Vec { + ReadableVec::collect(self) + } + + fn collect_holed(&self) -> Vec> { + ReadWriteRawVec::collect_holed(self) + } + + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option { + ReadWriteRawVec::get_with_reader(self, index, reader) + } + + fn reader(&self) -> Self::Reader { + ReadWriteRawVec::reader(self) + } + } + + // ============================================================================ + // Implementations for BytesVec + // ============================================================================ + + #[cfg(not(feature = "zerocopy"))] + use vecdb::ReadWriteRawVec; + use vecdb::{BytesStrategy, BytesVec}; + + impl RollbackVec for BytesVec { + fn import_with_changes<'a>( + db: &'a Database, + name: &'a str, + changes: u16, + ) -> Result<(Self, ImportOptions<'a>)> { + let mut options: ImportOptions = (db, name, Version::TWO).into(); + options = options.with_saved_stamped_changes(changes); + let vec = Self::forced_import_with(options)?; + Ok((vec, options)) + } + } + + impl RollbackOps for ReadWriteRawVec> { + type Reader = VecReader>; + + fn update(&mut self, index: usize, value: u32) -> Result<()> { + ReadWriteRawVec::update(self, index, value) + } + + fn take(&mut self, index: usize) -> Option { + let reader = self.reader(); + let result = ReadWriteRawVec::take(self, index, &reader); + drop(reader); + result + } + + fn stamped_write_with_changes(&mut self, stamp: Stamp) -> Result<()> { + WritableVec::stamped_write_with_changes(self, stamp) + } + + fn rollback(&mut self) -> Result<()> { + WritableVec::rollback(self) + } + + fn rollback_before(&mut self, stamp: Stamp) -> Result { + WritableVec::rollback_before(self, stamp) + } + + fn stamp(&self) -> Stamp { + AnyStoredVec::stamp(self) + } + + fn stored_len(&self) -> usize { + AnyStoredVec::stored_len(self) + } + + fn collect(&self) -> Vec { + ReadableVec::collect(self) + } + + fn collect_holed(&self) -> Vec> { + ReadWriteRawVec::collect_holed(self) + } + + fn get_with_reader(&self, index: usize, reader: &Self::Reader) -> Option { + ReadWriteRawVec::get_with_reader(self, index, reader) + } + + fn reader(&self) -> Self::Reader { + ReadWriteRawVec::reader(self) + } + } + + // ============================================================================ + // Generic Rollback Test Functions + // ============================================================================ + + fn run_basic_single_rollback() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Initial state: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + // Modify to [0, 1, 99, 3, 4] + vec.deref_mut().update(2, 99)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 99, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(2)); + + // Rollback to stamp 1 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_with_truncation() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Initial state: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + + // Add more: [0, 1, 2, 3, 4, 5, 6, 7] + vec.push(5); + vec.push(6); + vec.push(7); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4, 5, 6, 7]); + + // Rollback - should restore to [0, 1, 2, 3, 4] + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_multiple_sequential_rollbacks() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [0, 1, 2, 3, 4, 5] + vec.push(5); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + + // Stamp 3: [0, 1, 2, 3, 4, 5, 6] + vec.push(6); + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4, 5, 6]); + + // Rollback to stamp 2 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4, 5]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(2)); + + // Rollback to stamp 1 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_then_save_new_state() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [0, 1, 2, 3, 4, 5] + vec.push(5); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + + // Rollback to stamp 1 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + + // Now save a different state 2: [0, 1, 2, 3, 4, 99] + vec.push(99); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4, 99]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(2)); + + Ok(()) + } + + fn run_rollback_with_updates() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [0, 99, 2, 88, 4] - update multiple values + vec.deref_mut().update(1, 99)?; + vec.deref_mut().update(3, 88)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 99, 2, 88, 4]); + + // Rollback to stamp 1 - should restore original values + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_with_holes() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: delete some items (creating holes) + let _ = vec.deref_mut().take(1); + let _ = vec.deref_mut().take(3); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 2, 4]); + + // Rollback to stamp 1 - should restore deleted items + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_with_truncation_and_updates() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: extend + update + vec.deref_mut().update(1, 99)?; + vec.push(5); + vec.push(6); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 99, 2, 3, 4, 5, 6]); + + // Rollback - should restore length AND value + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_rollback_with_holes_and_updates() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: delete + update + let _ = vec.deref_mut().take(1); + vec.deref_mut().update(2, 99)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 99, 3, 4]); + + // Rollback - should restore deleted item AND original value + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + fn run_multiple_updates_to_same_index() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: [100, 1, 2, 3, 4] + vec.deref_mut().update(0, 100)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + + // Stamp 3: [200, 1, 2, 3, 4] + vec.deref_mut().update(0, 200)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + + // Stamp 4: [300, 1, 2, 3, 4] + vec.deref_mut().update(0, 300)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; + assert_eq!(vec.deref_mut().collect(), vec![300, 1, 2, 3, 4]); + + // Rollback to stamp 3 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![200, 1, 2, 3, 4]); + + // Rollback to stamp 2 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![100, 1, 2, 3, 4]); + + // Rollback to stamp 1 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + + Ok(()) + } + + fn run_complex_mixed_operations() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + for i in 0..10 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: Complex operations + // - Delete indices 1, 3, 5 + // - Update indices 2, 6, 8 + // - Push new values 100, 101 + let _ = vec.deref_mut().take(1); + let _ = vec.deref_mut().take(3); + let _ = vec.deref_mut().take(5); + vec.deref_mut().update(2, 222)?; + vec.deref_mut().update(6, 666)?; + vec.deref_mut().update(8, 888)?; + vec.push(100); + vec.push(101); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!( + vec.deref_mut().collect(), + vec![0, 222, 4, 666, 7, 888, 9, 100, 101] + ); + + // Rollback - should restore everything + vec.deref_mut().rollback()?; + assert_eq!( + vec.deref_mut().collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ); + + Ok(()) + } + + fn run_rollback_to_empty() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [] + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.deref_mut().collect(), Vec::::new()); + + // Stamp 2: [0, 1, 2] + vec.push(0); + vec.push(1); + vec.push(2); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2]); + + // Rollback to empty + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), Vec::::new()); + + Ok(()) + } + + fn run_reset() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Add initial data and flush + for i in 0..10 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.len(), 10); + assert_eq!(vec.deref_mut().stored_len(), 10); + assert_eq!(vec.pushed_len(), 0); + assert_eq!( + vec.deref_mut().collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ); + + // Add more data without flushing + vec.push(10); + vec.push(11); + assert_eq!(vec.len(), 12); + assert_eq!(vec.deref_mut().stored_len(), 10); + assert_eq!(vec.pushed_len(), 2); + + // Reset should clear everything + vec.reset()?; + assert_eq!(vec.len(), 0); + assert_eq!(vec.deref_mut().stored_len(), 0); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.deref_mut().collect(), Vec::::new()); + + // Should be able to add new data after reset + vec.push(100); + vec.push(101); + vec.push(102); + assert_eq!(vec.len(), 3); + assert_eq!(vec.deref_mut().stored_len(), 0); + assert_eq!(vec.pushed_len(), 3); + assert_eq!(vec.deref_mut().collect(), vec![100, 101, 102]); + + // Flush the new data + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.len(), 3); + assert_eq!(vec.deref_mut().stored_len(), 3); + assert_eq!(vec.pushed_len(), 0); + assert_eq!(vec.deref_mut().collect(), vec![100, 101, 102]); + + Ok(()) + } + + fn run_deep_rollback_chain() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Build a chain of 10 stamps with different operations + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; // [] + + vec.push(0); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; // [0] + + vec.push(1); + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; // [0, 1] + + vec.deref_mut().update(0, 10)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; // [10, 1] + + vec.push(2); + vec.deref_mut().stamped_write_with_changes(Stamp::new(5))?; // [10, 1, 2] + + let _ = vec.deref_mut().take(1); + vec.deref_mut().stamped_write_with_changes(Stamp::new(6))?; // [10, 2] + + vec.push(3); + vec.deref_mut().stamped_write_with_changes(Stamp::new(7))?; // [10, 2, 3] + + vec.deref_mut().update(0, 20)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(8))?; // [20, 2, 3] + + vec.push(4); + vec.push(5); + vec.deref_mut().stamped_write_with_changes(Stamp::new(9))?; // [20, 2, 3, 4, 5] + + vec.deref_mut().update(2, 33)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(10))?; // [20, 33, 3, 4, 5] + assert_eq!(vec.deref_mut().collect(), vec![20, 33, 3, 4, 5]); + + // Rollback through the chain + vec.deref_mut().rollback()?; // -> 9 + assert_eq!(vec.deref_mut().collect(), vec![20, 2, 3, 4, 5]); + + vec.deref_mut().rollback()?; // -> 8 + assert_eq!(vec.deref_mut().collect(), vec![20, 2, 3]); + + vec.deref_mut().rollback()?; // -> 7 + assert_eq!(vec.deref_mut().collect(), vec![10, 2, 3]); + + vec.deref_mut().rollback()?; // -> 6 + assert_eq!(vec.deref_mut().collect(), vec![10, 2]); + + vec.deref_mut().rollback()?; // -> 5 + assert_eq!(vec.deref_mut().collect(), vec![10, 1, 2]); + + vec.deref_mut().rollback()?; // -> 4 + assert_eq!(vec.deref_mut().collect(), vec![10, 1]); + + vec.deref_mut().rollback()?; // -> 3 + assert_eq!(vec.deref_mut().collect(), vec![0, 1]); + + vec.deref_mut().rollback()?; // -> 2 + assert_eq!(vec.deref_mut().collect(), vec![0]); + + vec.deref_mut().rollback()?; // -> 1 + assert_eq!(vec.deref_mut().collect(), Vec::::new()); + + Ok(()) + } + + fn run_rollback_all_elements_updated() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4] + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: Update ALL elements + for i in 0..5 { + vec.deref_mut().update(i, (i * 100) as u32)?; + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 100, 200, 300, 400]); + + // Rollback - should restore all original values + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4]); + + Ok(()) + } + + fn run_multiple_holes_then_rollback() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + for i in 0..10 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Stamp 2: Delete every other element + for i in (0..10).step_by(2) { + let _ = vec.deref_mut().take(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![1, 3, 5, 7, 9]); + + // Rollback - should restore all deleted items + vec.deref_mut().rollback()?; + assert_eq!( + vec.deref_mut().collect(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ); + + Ok(()) + } + + fn run_rollback_before() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Build stamps 1-5 + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + vec.push(5); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + + vec.push(6); + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + + vec.push(7); + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; + + vec.push(8); + vec.deref_mut().stamped_write_with_changes(Stamp::new(5))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8]); + + // Rollback before stamp 4 (should go to stamp 3) + let _ = vec.deref_mut().rollback_before(Stamp::new(4))?; + assert_eq!(vec.deref_mut().collect(), vec![0, 1, 2, 3, 4, 5, 6]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(3)); + + Ok(()) + } + + /// Regression test: rollback-after-rollback with delete_at losing entries. + /// + /// After the first rollback, restored entries sit in `updated.current`. + /// If `delete_at` removes one from `updated.current` during reprocessing, + /// and `serialize_changes` only iterated `updated.current` keys (the old bug), + /// the entry's prev value would be lost from the change file. + /// On a second rollback, the slot would contain stale on-disk data + /// instead of the correct rolled-back value. + fn run_rollback_after_rollback_with_delete() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + let (db, _temp) = setup_db()?; + let (mut vec, _) = V::import_with_changes(&db, "test", 10)?; + + // Stamp 1 (baseline): [10, 20, 30, 40, 50] + for &v in &[10, 20, 30, 40, 50] { + vec.push(v); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.deref_mut().collect(), vec![10, 20, 30, 40, 50]); + + // Stamp 2: update slot 2 (30 → 99), delete slot 1 (creates hole) + vec.deref_mut().update(2, 99)?; + let _ = vec.deref_mut().take(1); + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.deref_mut().collect(), vec![10, 99, 40, 50]); + + // First rollback → back to stamp 1 + vec.deref_mut().rollback()?; + assert_eq!(vec.deref_mut().collect(), vec![10, 20, 30, 40, 50]); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + // Now reprocess: delete slot 2 (the one we just restored), update slot 3 + // This simulates an address becoming empty during reprocessing + let _ = vec.deref_mut().take(2); // removes 30 from updated.current + vec.deref_mut().update(3, 88)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + assert_eq!(vec.deref_mut().collect(), vec![10, 20, 88, 50]); + + // Second rollback → must go back to stamp 1 values + vec.deref_mut().rollback()?; + let result = vec.deref_mut().collect(); + assert_eq!( + result, + vec![10, 20, 30, 40, 50], + "Second rollback must restore all original values. \ + Slot 2 was deleted during reprocessing but its prev value (30) \ + must still be tracked in the change file." + ); + assert_eq!(vec.deref_mut().stamp(), Stamp::new(1)); + + Ok(()) + } + + // ============================================================================ + // Test instantiation for each raw vec type + // ============================================================================ + + #[cfg(feature = "zerocopy")] + mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn basic_single_rollback() -> Result<()> { + run_basic_single_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_with_updates() -> Result<()> { + run_rollback_with_updates::() + } + #[test] + fn rollback_with_holes() -> Result<()> { + run_rollback_with_holes::() + } + #[test] + fn rollback_with_truncation_and_updates() -> Result<()> { + run_rollback_with_truncation_and_updates::() + } + #[test] + fn rollback_with_holes_and_updates() -> Result<()> { + run_rollback_with_holes_and_updates::() + } + #[test] + fn multiple_updates_to_same_index() -> Result<()> { + run_multiple_updates_to_same_index::() + } + #[test] + fn complex_mixed_operations() -> Result<()> { + run_complex_mixed_operations::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_all_elements_updated() -> Result<()> { + run_rollback_all_elements_updated::() + } + #[test] + fn multiple_holes_then_rollback() -> Result<()> { + run_multiple_holes_then_rollback::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn rollback_after_rollback_with_delete() -> Result<()> { + run_rollback_after_rollback_with_delete::() + } + } + + mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn basic_single_rollback() -> Result<()> { + run_basic_single_rollback::() + } + #[test] + fn rollback_with_truncation() -> Result<()> { + run_rollback_with_truncation::() + } + #[test] + fn multiple_sequential_rollbacks() -> Result<()> { + run_multiple_sequential_rollbacks::() + } + #[test] + fn rollback_then_save_new_state() -> Result<()> { + run_rollback_then_save_new_state::() + } + #[test] + fn rollback_with_updates() -> Result<()> { + run_rollback_with_updates::() + } + #[test] + fn rollback_with_holes() -> Result<()> { + run_rollback_with_holes::() + } + #[test] + fn rollback_with_truncation_and_updates() -> Result<()> { + run_rollback_with_truncation_and_updates::() + } + #[test] + fn rollback_with_holes_and_updates() -> Result<()> { + run_rollback_with_holes_and_updates::() + } + #[test] + fn multiple_updates_to_same_index() -> Result<()> { + run_multiple_updates_to_same_index::() + } + #[test] + fn complex_mixed_operations() -> Result<()> { + run_complex_mixed_operations::() + } + #[test] + fn rollback_to_empty() -> Result<()> { + run_rollback_to_empty::() + } + #[test] + fn deep_rollback_chain() -> Result<()> { + run_deep_rollback_chain::() + } + #[test] + fn rollback_all_elements_updated() -> Result<()> { + run_rollback_all_elements_updated::() + } + #[test] + fn multiple_holes_then_rollback() -> Result<()> { + run_multiple_holes_then_rollback::() + } + #[test] + fn rollback_before() -> Result<()> { + run_rollback_before::() + } + #[test] + fn reset() -> Result<()> { + run_reset::() + } + #[test] + fn rollback_after_rollback_with_delete() -> Result<()> { + run_rollback_after_rollback_with_delete::() + } + } +} // end mod raw_rollback + +// ============================================================================ +// PART 3: Comprehensive Integration Test +// ============================================================================ +// Complex rollback + flush + reopen test with file integrity verification. + +mod integration { + use crate::raw_rollback::{RollbackOps, RollbackVec}; + + use super::*; + use sha2::{Digest, Sha256}; + use std::fs; + use std::path::Path; + + /// Compute SHA-256 hash of the vecdb data file and regions directory + /// Only hashes data (file) and regions/*, ignoring changes directory + fn compute_directory_hash(dir: &Path) -> Result { + use std::path::PathBuf; + + let mut hasher = Sha256::new(); + + // Collect all files in sorted order for deterministic hashing + let mut files: Vec = Vec::new(); + + // Hash the data file if it exists + let data_file = dir.join("data"); + if data_file.exists() && data_file.is_file() { + files.push(data_file); + } + + // Hash files in the regions directory, excluding changes subdirectory + let regions_dir = dir.join("regions"); + if regions_dir.exists() { + fn collect_files(dir: &Path, files: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.components().any(|c| c.as_os_str() == "changes") { + continue; + } + if path.is_dir() { + collect_files(&path, files); + } else if path.is_file() { + files.push(path); + } + } + } + collect_files(®ions_dir, &mut files); + } + + files.sort(); + + // Hash each file's relative path and contents + for file_path in &files { + // Hash the relative path + if let Ok(rel_path) = file_path.strip_prefix(dir) { + hasher.update(rel_path.to_string_lossy().as_bytes()); + } + + // Hash the file contents + let contents = fs::read(file_path)?; + hasher.update(&contents); + } + + let hash = hasher.finalize(); + Ok(hash.iter().map(|b| format!("{:02x}", b)).collect()) + } + + /// Comprehensive integration test: rollback + flush + reopen with integrity verification. + /// + /// This test verifies that after rollback + flush + close + reopen: + /// 1. Data can be correctly read back using individual gets + /// 2. Data can be correctly read back using iterators + /// 3. Redo operations produce the same readable state + fn run_data_integrity_rollback_flush_reopen() -> Result<()> + where + V: RollbackVec, + V::Target: RollbackOps, + { + // Create database + let (database, temp) = setup_db()?; + let test_path = temp.path(); + + let (mut vec, _) = V::import_with_changes(&database, "vec", 10)?; + + // Phase 1: Initial work + for i in 0..5 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(1))?; + + // Phase 2: More work + for i in 5..10 { + vec.push(i); + } + vec.deref_mut().stamped_write_with_changes(Stamp::new(2))?; + + // Checkpoint 1 + let checkpoint1_data = vec.deref_mut().collect_holed(); + let checkpoint1_stamp = vec.deref_mut().stamp(); + let _checkpoint1_hash = compute_directory_hash(test_path)?; + + // Phase 3: Three more operations with flush + vec.deref_mut().update(2, 100)?; + vec.deref_mut().update(7, 200)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + + vec.push(20); + vec.push(21); + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; + + let _ = vec.deref_mut().take(5); + vec.push(30); + vec.deref_mut().stamped_write_with_changes(Stamp::new(5))?; + + // Checkpoint 2 + let checkpoint2_data = vec.deref_mut().collect_holed(); + let checkpoint2_stamp = vec.deref_mut().stamp(); + let _checkpoint2_hash = compute_directory_hash(test_path)?; + + // Undo last 3 operations + vec.deref_mut().rollback()?; + vec.deref_mut().rollback()?; + vec.deref_mut().rollback()?; + + // Verify in-memory data matches checkpoint1 + let after_undo_data = vec.deref_mut().collect_holed(); + let after_undo_stamp = vec.deref_mut().stamp(); + + assert_eq!(after_undo_stamp, checkpoint1_stamp); + assert_eq!(after_undo_data, checkpoint1_data); + + // Flush and close + vec.deref_mut() + .stamped_write_with_changes(checkpoint1_stamp)?; + let _after_flush_hash = compute_directory_hash(test_path)?; + + drop(vec); + + // Reopen + let (mut vec, _) = V::import_with_changes(&database, "vec", 10)?; + + // Verify using individual gets + let reader = vec.deref_mut().reader(); + let mut data_via_gets = Vec::new(); + for i in 0..vec.len() { + let value = vec.get_with_reader(i, &reader); + data_via_gets.push(value); + } + drop(reader); + + assert_eq!(data_via_gets, checkpoint1_data); + + // Verify using iterator + let data_via_iter = vec.deref_mut().collect_holed(); + assert_eq!(data_via_iter, checkpoint1_data); + + // Redo the same 3 operations + vec.deref_mut().update(2, 100)?; + vec.deref_mut().update(7, 200)?; + vec.deref_mut().stamped_write_with_changes(Stamp::new(3))?; + + vec.push(20); + vec.push(21); + vec.deref_mut().stamped_write_with_changes(Stamp::new(4))?; + + let _ = vec.deref_mut().take(5); + vec.push(30); + vec.deref_mut().stamped_write_with_changes(Stamp::new(5))?; + + // Verify in-memory data matches checkpoint2 + let after_redo_data = vec.deref_mut().collect_holed(); + let after_redo_stamp = vec.deref_mut().stamp(); + + assert_eq!(after_redo_stamp, checkpoint2_stamp); + assert_eq!(after_redo_data, checkpoint2_data); + + // Flush and close + vec.deref_mut() + .stamped_write_with_changes(checkpoint2_stamp)?; + drop(vec); + + // Reopen again + let (vec, _) = V::import_with_changes(&database, "vec", 10)?; + + // Verify using individual gets + let reader = vec.deref().reader(); + let mut data_via_gets = Vec::new(); + for i in 0..vec.len() { + let value = vec.get_with_reader(i, &reader); + data_via_gets.push(value); + } + drop(reader); + + assert_eq!(data_via_gets, checkpoint2_data); + + // Verify using iterator + let data_via_iter = vec.deref().collect_holed(); + assert_eq!(data_via_iter, checkpoint2_data); + + Ok(()) + } + + #[cfg(feature = "zerocopy")] + mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn data_integrity_rollback_flush_reopen() -> Result<()> { + run_data_integrity_rollback_flush_reopen::() + } + } + + mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn data_integrity_rollback_flush_reopen() -> Result<()> { + run_data_integrity_rollback_flush_reopen::() + } + } +} diff --git a/crates/vecdb/tests/rollback_truncation.rs b/crates/vecdb/tests/rollback_truncation.rs new file mode 100644 index 000000000..c855b797b --- /dev/null +++ b/crates/vecdb/tests/rollback_truncation.rs @@ -0,0 +1,179 @@ +//! Rollback of stamps that included a `truncate_if_needed_at` shrink. +//! (The `rollback_with_truncation` test in `tests/rollback.rs` is misnamed — +//! it never actually calls `truncate_if_needed_at`.) + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ImportOptions, Result, Stamp, StoredVec, Version}; + +fn setup_db() -> Result<(Database, TempDir)> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + Ok((db, temp)) +} + +fn import_with_changes(db: &Database, name: &str, changes: u16) -> Result +where + V: StoredVec, +{ + let mut options: ImportOptions = (db, name, Version::TWO).into(); + options = options.with_saved_stamped_changes(changes); + V::forced_import_with(options) +} + +fn run_rollback_after_pure_truncate() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "pure_truncate", 10)?; + + for i in 0..8 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + assert_eq!(vec.len(), 8); + assert_eq!(vec.collect(), (0..8).collect::>()); + + vec.truncate_if_needed_at(5)?; + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.len(), 5); + assert_eq!(vec.collect(), (0..5).collect::>()); + + vec.rollback()?; + assert_eq!(vec.len(), 8, "len mismatch after rollback of truncation"); + assert_eq!( + vec.collect(), + (0..8).collect::>(), + "data mismatch after rollback of truncation" + ); + assert_eq!(vec.stamp(), Stamp::new(1)); + + Ok(()) +} + +fn run_rollback_after_truncate_and_push() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "truncate_push", 10)?; + + for i in 0..5 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + vec.truncate_if_needed_at(3)?; + vec.push(100); + vec.push(200); + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.len(), 5); + assert_eq!(vec.collect(), vec![0, 1, 2, 100, 200]); + + vec.rollback()?; + assert_eq!(vec.len(), 5, "len mismatch after rollback"); + assert_eq!( + vec.collect(), + vec![0, 1, 2, 3, 4], + "data mismatch after rollback" + ); + assert_eq!(vec.stamp(), Stamp::new(1)); + + Ok(()) +} + +fn run_rollback_truncate_then_reflush() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + let mut vec = import_with_changes::(&db, "truncate_reflush", 10)?; + + for i in 0..10 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + vec.truncate_if_needed_at(4)?; + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.len(), 4); + + vec.rollback()?; + assert_eq!(vec.len(), 10); + assert_eq!(vec.collect(), (0..10).collect::>()); + + vec.stamped_write_with_changes(Stamp::new(2))?; + assert_eq!(vec.len(), 10); + assert_eq!(vec.stored_len(), 10); + assert_eq!(vec.collect(), (0..10).collect::>()); + + Ok(()) +} + +fn run_rollback_truncate_persistence() -> Result<()> +where + V: StoredVec, +{ + let (db, _temp) = setup_db()?; + { + let mut vec = import_with_changes::(&db, "persist", 10)?; + for i in 0..7 { + vec.push(i); + } + vec.stamped_write_with_changes(Stamp::new(1))?; + + vec.truncate_if_needed_at(2)?; + vec.stamped_write_with_changes(Stamp::new(2))?; + + vec.rollback()?; + vec.stamped_write_with_changes(Stamp::new(1))?; + } + { + let vec = import_with_changes::(&db, "persist", 10)?; + assert_eq!(vec.len(), 7); + assert_eq!(vec.collect(), (0..7).collect::>()); + assert_eq!(vec.stamp(), Stamp::new(1)); + } + Ok(()) +} + +macro_rules! instantiate_for { + ($mod:ident, $ty:ty) => { + mod $mod { + use super::*; + type V = $ty; + + #[test] + fn rollback_after_pure_truncate() -> Result<()> { + run_rollback_after_pure_truncate::() + } + #[test] + fn rollback_after_truncate_and_push() -> Result<()> { + run_rollback_after_truncate_and_push::() + } + #[test] + fn rollback_truncate_then_reflush() -> Result<()> { + run_rollback_truncate_then_reflush::() + } + #[test] + fn rollback_truncate_persistence() -> Result<()> { + run_rollback_truncate_persistence::() + } + } + }; +} + +instantiate_for!(bytes, vecdb::BytesVec); + +#[cfg(feature = "zerocopy")] +instantiate_for!(zerocopy, vecdb::ZeroCopyVec); + +#[cfg(feature = "pco")] +instantiate_for!(pco, vecdb::PcoVec); + +#[cfg(feature = "lz4")] +instantiate_for!(lz4, vecdb::LZ4Vec); + +#[cfg(feature = "zstd")] +instantiate_for!(zstd, vecdb::ZstdVec); diff --git a/crates/vecdb/tests/scaling_bench.rs b/crates/vecdb/tests/scaling_bench.rs new file mode 100644 index 000000000..468da5590 --- /dev/null +++ b/crates/vecdb/tests/scaling_bench.rs @@ -0,0 +1,256 @@ +//! Scaling benchmark to find maximum concurrent read/write performance. +//! +//! Run with: cargo test --package vecdb --test scaling_bench --features pco --release -- --ignored --nocapture + +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + thread, + time::{Duration, Instant}, +}; + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, AnyVec, ImportableVec, ReadableVec, Result, StoredVec, Version, WritableVec, +}; + +#[cfg(feature = "pco")] +use vecdb::PcoVec; + +fn setup_test_db() -> Result<(Database, TempDir)> { + let temp_dir = TempDir::new()?; + let db = Database::open(temp_dir.path())?; + Ok((db, temp_dir)) +} + +/// Test configuration +#[cfg(feature = "pco")] +struct Config { + num_readers: usize, + write_interval_us: u64, + batch_size: usize, + duration_secs: u64, +} + +/// Run a single benchmark with the given configuration +#[cfg(feature = "pco")] +fn run_benchmark(config: &Config) -> Result<(usize, usize, usize)> { + let (db, _temp) = setup_test_db()?; + let version = Version::ONE; + + let mut writer: PcoVec = PcoVec::forced_import(&db, "bench_vec", version)?; + + // Write initial data + for i in 0..1000u64 { + writer.push(i); + } + writer.write()?; + + let stop = Arc::new(AtomicBool::new(false)); + let reads_completed = Arc::new(AtomicUsize::new(0)); + let read_errors = Arc::new(AtomicUsize::new(0)); + + // Spawn readers + let reader_handles: Vec<_> = (0..config.num_readers) + .map(|_| { + let reader = writer.read_only_clone(); + let stop = stop.clone(); + let reads = reads_completed.clone(); + let errs = read_errors.clone(); + + thread::spawn(move || { + let mut local_reads = 0usize; + let mut local_errors = 0usize; + + while !stop.load(Ordering::Relaxed) { + let len = reader.len(); + if len > 0 { + let idx = len - 1; + if let Some(v) = reader.collect_one(idx) { + if v != idx as u64 { + eprintln!( + "ERROR: idx={} expected={} got={} len={}", + idx, idx, v, len + ); + local_errors += 1; + } + local_reads += 1; + } else { + eprintln!("ERROR: empty collect at idx={} len={}", idx, len); + local_errors += 1; + } + } + // Tight loop - no sleep + } + + reads.fetch_add(local_reads, Ordering::Relaxed); + errs.fetch_add(local_errors, Ordering::Relaxed); + }) + }) + .collect(); + + // Writer + let start = Instant::now(); + let target_duration = Duration::from_secs(config.duration_secs); + let mut current_idx = 1000u64; + let mut writes = 0usize; + + while start.elapsed() < target_duration { + for _ in 0..config.batch_size { + writer.push(current_idx); + current_idx += 1; + } + writer.write()?; + writes += 1; + + if config.write_interval_us > 0 { + thread::sleep(Duration::from_micros(config.write_interval_us)); + } + } + + // Stop readers + stop.store(true, Ordering::Relaxed); + thread::sleep(Duration::from_millis(50)); + + for handle in reader_handles { + handle.join().unwrap(); + } + + let final_reads = reads_completed.load(Ordering::Relaxed); + let final_errors = read_errors.load(Ordering::Relaxed); + + Ok((writes, final_reads, final_errors)) +} + +#[test] +#[ignore] +#[cfg(feature = "pco")] +fn test_scale_readers() -> Result<()> { + println!("\n=== Scaling Readers (fixed write interval: 10ms, batch: 100) ===\n"); + println!( + "{:>8} {:>12} {:>12} {:>8}", + "Readers", "Writes", "Reads", "Errors" + ); + println!("{:-<44}", ""); + + for num_readers in [1, 2, 4, 8, 16, 32, 64] { + let config = Config { + num_readers, + write_interval_us: 10_000, // 10ms + batch_size: 100, + duration_secs: 3, + }; + + let (writes, reads, errors) = run_benchmark(&config)?; + println!( + "{:>8} {:>12} {:>12} {:>8}", + num_readers, writes, reads, errors + ); + + if errors > 0 { + println!(">>> ERRORS DETECTED - stopping scale test"); + break; + } + } + + Ok(()) +} + +#[test] +#[ignore] +#[cfg(feature = "pco")] +fn test_scale_write_frequency() -> Result<()> { + println!("\n=== Scaling Write Frequency (fixed readers: 4, batch: 100) ===\n"); + println!( + "{:>12} {:>12} {:>12} {:>8}", + "Interval(us)", "Writes", "Reads", "Errors" + ); + println!("{:-<48}", ""); + + for write_interval_us in [100_000, 50_000, 10_000, 5_000, 1_000, 500, 100, 0] { + let config = Config { + num_readers: 4, + write_interval_us, + batch_size: 100, + duration_secs: 3, + }; + + let (writes, reads, errors) = run_benchmark(&config)?; + println!( + "{:>12} {:>12} {:>12} {:>8}", + write_interval_us, writes, reads, errors + ); + + if errors > 0 { + println!(">>> ERRORS DETECTED - stopping scale test"); + break; + } + } + + Ok(()) +} + +#[test] +#[ignore] +#[cfg(feature = "pco")] +fn test_scale_batch_size() -> Result<()> { + println!("\n=== Scaling Batch Size (fixed readers: 4, interval: 1ms) ===\n"); + println!( + "{:>12} {:>12} {:>12} {:>8}", + "Batch Size", "Writes", "Reads", "Errors" + ); + println!("{:-<48}", ""); + + for batch_size in [10, 50, 100, 500, 1000, 5000, 10000] { + let config = Config { + num_readers: 4, + write_interval_us: 1_000, // 1ms + batch_size, + duration_secs: 3, + }; + + let (writes, reads, errors) = run_benchmark(&config)?; + println!( + "{:>12} {:>12} {:>12} {:>8}", + batch_size, writes, reads, errors + ); + + if errors > 0 { + println!(">>> ERRORS DETECTED - stopping scale test"); + break; + } + } + + Ok(()) +} + +#[test] +#[ignore] +#[cfg(feature = "pco")] +fn test_extreme_stress() -> Result<()> { + println!("\n=== Extreme Stress Test ===\n"); + println!("64 readers, no write delay, batch size 100, 10 seconds\n"); + + let config = Config { + num_readers: 64, + write_interval_us: 0, + batch_size: 100, + duration_secs: 10, + }; + + let (writes, reads, errors) = run_benchmark(&config)?; + + println!("Results:"); + println!(" Writes: {}", writes); + println!(" Reads: {}", reads); + println!(" Errors: {}", errors); + println!(" Reads/sec: {:.0}", reads as f64 / 10.0); + println!(" Writes/sec: {:.0}", writes as f64 / 10.0); + + assert_eq!(errors, 0, "Data integrity errors detected!"); + + Ok(()) +} diff --git a/crates/vecdb/tests/write_visibility.rs b/crates/vecdb/tests/write_visibility.rs new file mode 100644 index 000000000..bef40414b --- /dev/null +++ b/crates/vecdb/tests/write_visibility.rs @@ -0,0 +1,272 @@ +//! Tests that write() makes data visible to other vecs without fsync. +//! +//! This verifies the key property that enables fast per-compute operations: +//! after vec_a.write(), vec_b can immediately read the new data because +//! both share the same mmap. No fsync is needed for in-process visibility. + +use rawdb::Database; +use tempfile::TempDir; +use vecdb::{Result, StoredVec, Version}; + +fn setup_test_db() -> Result<(Database, TempDir)> { + let temp_dir = TempDir::new()?; + let db = Database::open(temp_dir.path())?; + Ok((db, temp_dir)) +} + +/// Tests that after write() on vec_a, a separate vec_b instance +/// can read the new data (simulating derived vec computation). +fn run_write_visibility_test() -> Result<(), Box> +where + V: StoredVec, +{ + let version = Version::ZERO; + let (database, _temp) = setup_test_db()?; + + // Create and populate vec_a + let mut vec_a: V = V::forced_import(&database, "vec_a", version)?; + for i in 0..100u32 { + vec_a.push(i); + } + + // Write without flush - just mmap, no fsync + vec_a.write()?; + + // Now create vec_b that will read from vec_a + // This simulates the pattern: compute vec_b derived from vec_a + let vec_a_reader: V = V::forced_import(&database, "vec_a", version)?; + + // vec_b should see all 100 values written by vec_a + assert_eq!( + vec_a_reader.len(), + 100, + "vec_b should see vec_a's written data" + ); + + for i in 0..100usize { + assert_eq!( + vec_a_reader.collect_one(i), + Some(i as u32), + "vec_b should read correct value at index {}", + i + ); + } + + Ok(()) +} + +/// Tests the full compute chain pattern: +/// 1. Compute and write vec_a +/// 2. Compute vec_b derived from vec_a, write it +/// 3. Compute vec_c derived from vec_b, write it +/// 4. Final flush for durability +fn run_compute_chain_test() -> Result<(), Box> +where + V: StoredVec, +{ + let version = Version::ZERO; + let (database, _temp) = setup_test_db()?; + + // Step 1: Compute vec_a (source data) + let mut vec_a: V = V::forced_import(&database, "chain_a", version)?; + for i in 0..50u32 { + vec_a.push(i * 2); // Even numbers: 0, 2, 4, ... + } + vec_a.write()?; // No fsync, just mmap write + + // Step 2: Compute vec_b derived from vec_a (sum of consecutive pairs) + let vec_a_for_read: V = V::forced_import(&database, "chain_a", version)?; + let mut vec_b: V = V::forced_import(&database, "chain_b", version)?; + + for i in 0..25usize { + let a1 = vec_a_for_read.collect_one(i * 2).unwrap(); + let a2 = vec_a_for_read.collect_one(i * 2 + 1).unwrap(); + vec_b.push(a1 + a2); + } + vec_b.write()?; // No fsync, just mmap write + + // Step 3: Compute vec_c derived from vec_b (cumulative sum) + let vec_b_for_read: V = V::forced_import(&database, "chain_b", version)?; + let mut vec_c: V = V::forced_import(&database, "chain_c", version)?; + + let mut cumsum = 0u32; + for val in vec_b_for_read.collect() { + cumsum += val; + vec_c.push(cumsum); + } + vec_c.write()?; // No fsync, just mmap write + + // Verify the chain computed correctly + // vec_a: [0, 2, 4, 6, 8, 10, ...] + // vec_b: [0+2, 4+6, 8+10, ...] = [2, 10, 18, ...] + // vec_c: cumsum of vec_b + + let vec_c_verify: V = V::forced_import(&database, "chain_c", version)?; + assert_eq!(vec_c_verify.len(), 25); + + let expected_b: Vec = (0..25).map(|i| (i * 4) + (i * 4 + 2)).collect(); + let expected_c: Vec = expected_b + .iter() + .scan(0u32, |acc, &x| { + *acc += x; + Some(*acc) + }) + .collect(); + + let actual_c: Vec = vec_c_verify.collect(); + assert_eq!( + actual_c, expected_c, + "compute chain should produce correct results" + ); + + Ok(()) +} + +/// Tests that write() returns the correct boolean: +/// - true if data was written +/// - false if nothing to write +fn run_write_returns_bool_test() -> Result<(), Box> +where + V: StoredVec, +{ + let version = Version::ZERO; + let (database, _temp) = setup_test_db()?; + + let mut vec: V = V::forced_import(&database, "bool_test", version)?; + + // First write with no data should return false + assert!(!vec.write()?, "write() with no data should return false"); + + // Push data + vec.push(42); + + // Write with data should return true + assert!(vec.write()?, "write() with data should return true"); + + // Second write with no new data should return false + assert!( + !vec.write()?, + "write() after already written should return false" + ); + + // Push more data + vec.push(43); + + // Write should return true again + assert!(vec.write()?, "write() with new data should return true"); + + Ok(()) +} + +// ============================================================================ +// Test instantiation for each vec type +// ============================================================================ + +mod bytes { + use super::*; + use vecdb::BytesVec; + type V = BytesVec; + + #[test] + fn test_write_visibility() -> Result<(), Box> { + run_write_visibility_test::() + } + + #[test] + fn test_compute_chain() -> Result<(), Box> { + run_compute_chain_test::() + } + + #[test] + fn test_write_returns_bool() -> Result<(), Box> { + run_write_returns_bool_test::() + } +} + +#[cfg(feature = "pco")] +mod pco { + use super::*; + use vecdb::PcoVec; + type V = PcoVec; + + #[test] + fn test_write_visibility() -> Result<(), Box> { + run_write_visibility_test::() + } + + #[test] + fn test_compute_chain() -> Result<(), Box> { + run_compute_chain_test::() + } + + #[test] + fn test_write_returns_bool() -> Result<(), Box> { + run_write_returns_bool_test::() + } +} + +#[cfg(feature = "lz4")] +mod lz4 { + use super::*; + use vecdb::LZ4Vec; + type V = LZ4Vec; + + #[test] + fn test_write_visibility() -> Result<(), Box> { + run_write_visibility_test::() + } + + #[test] + fn test_compute_chain() -> Result<(), Box> { + run_compute_chain_test::() + } + + #[test] + fn test_write_returns_bool() -> Result<(), Box> { + run_write_returns_bool_test::() + } +} + +#[cfg(feature = "zstd")] +mod zstd { + use super::*; + use vecdb::ZstdVec; + type V = ZstdVec; + + #[test] + fn test_write_visibility() -> Result<(), Box> { + run_write_visibility_test::() + } + + #[test] + fn test_compute_chain() -> Result<(), Box> { + run_compute_chain_test::() + } + + #[test] + fn test_write_returns_bool() -> Result<(), Box> { + run_write_returns_bool_test::() + } +} + +#[cfg(feature = "zerocopy")] +mod zerocopy { + use super::*; + use vecdb::ZeroCopyVec; + type V = ZeroCopyVec; + + #[test] + fn test_write_visibility() -> Result<(), Box> { + run_write_visibility_test::() + } + + #[test] + fn test_compute_chain() -> Result<(), Box> { + run_compute_chain_test::() + } + + #[test] + fn test_write_returns_bool() -> Result<(), Box> { + run_write_returns_bool_test::() + } +} diff --git a/crates/vecdb_derive/Cargo.toml b/crates/vecdb_derive/Cargo.toml new file mode 100644 index 000000000..b6eda1a85 --- /dev/null +++ b/crates/vecdb_derive/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "vecdb_derive" +description = "Derive for vecdb" +keywords = ["vec", "disk", "data"] +categories = ["database"] +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +readme.workspace = true + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" + +[dev-dependencies] +vecdb = { workspace = true, features = ["derive", "pco"] } +tempfile = { workspace = true } diff --git a/crates/vecdb_derive/README.md b/crates/vecdb_derive/README.md new file mode 100644 index 000000000..72926549c --- /dev/null +++ b/crates/vecdb_derive/README.md @@ -0,0 +1,133 @@ +# vecdb_derive + +Derive macros for [`vecdb`](../vecdb/) to enable custom types in compressed and uncompressed vectors. + +Provides two derive macros: +- `#[derive(Bytes)]` - For use with `BytesVec`, `LZ4Vec`, `ZstdVec` +- `#[derive(Pco)]` - For use with `PcoVec` (compressed numeric vectors) + +## Install + +```bash +cargo add vecdb --features derive +``` + +## Usage + +### Bytes Derive + +Use `#[derive(Bytes)]` to enable custom wrapper types with uncompressed or general-purpose compressed vectors: + +```rust +use vecdb::{Bytes, BytesVec, Database, Importable, Version}; + +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct UserId(u64); + +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct Timestamp(i64); + +fn main() -> vecdb::Result<()> { + let db = Database::open("data")?; + + // Works with BytesVec + let mut users: BytesVec = + BytesVec::import(&db, "users", Version::TWO)?; + users.push(UserId(12345)); + users.flush()?; + + // Also works with LZ4Vec and ZstdVec + let mut timestamps: vecdb::LZ4Vec = + vecdb::LZ4Vec::import(&db, "timestamps", Version::TWO)?; + timestamps.push(Timestamp(1700000000)); + timestamps.flush()?; + + Ok(()) +} +``` + +**Requirements:** +- Must be a tuple struct with exactly one field +- Inner type must implement `Bytes` +- Works with generic types + +### Pco Derive + +Use `#[derive(Pco)]` for numeric wrapper types to enable Pcodec compression: + +```rust +use vecdb::{Pco, PcoVec, Database, Importable, Version}; + +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Price(f64); + +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Quantity(u32); + +fn main() -> vecdb::Result<()> { + let db = Database::open("data")?; + + let mut prices: PcoVec = + PcoVec::import(&db, "prices", Version::TWO)?; + prices.push(Price(99.99)); + prices.flush()?; + + Ok(()) +} +``` + +**Requirements:** +- Must be a tuple struct with exactly one field +- Inner type must implement `Pco` (numeric types only: u16-u64, i16-i64, f32, f64) +- The derive automatically implements both `Bytes` and `Pco` traits +- Works with generic types + +### Generic Types + +Both derives support generic type parameters: + +```rust +use vecdb::{Bytes, Pco}; + +// Generic wrapper with Bytes +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct Wrapper(T); + +// Generic wrapper with Pco +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct NumericWrapper(T); + +// Nested generics +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Container(Wrapper); + +// Use with concrete types +let value = Wrapper(42u64); +let numeric = NumericWrapper(3.14f64); +let nested = Container(Wrapper(100u32)); +``` + +The derives automatically add appropriate trait bounds (`T: Bytes` or `T: Pco`) to the generated implementations. + +## How It Works + +Both derives implement traits by delegating to the inner type: + +```rust +// #[derive(Bytes)] generates: +impl Bytes for Wrapper where T: Bytes { + fn to_bytes(&self) -> Vec { + self.0.to_bytes() + } + fn from_bytes(bytes: &[u8]) -> Result { + Ok(Self(T::from_bytes(bytes)?)) + } +} + +// #[derive(Pco)] generates: +// - Bytes implementation (same as above) +// - Pco implementation with NumberType from inner type +// - TransparentPco marker trait +``` + +This allows wrapper types to have the same serialization and compression characteristics as their inner type. diff --git a/crates/vecdb_derive/src/lib.rs b/crates/vecdb_derive/src/lib.rs new file mode 100644 index 000000000..cec3dcb1b --- /dev/null +++ b/crates/vecdb_derive/src/lib.rs @@ -0,0 +1,251 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{Data, DataStruct, DeriveInput, Fields, parse_macro_input}; + +/// Derives the `Bytes` trait for single-field tuple structs. +/// +/// This macro enables custom wrapper types to work with `BytesVec`, `LZ4Vec`, `ZstdVec`, +/// and other vecdb vector types that require the `Bytes` trait. +/// +/// # Requirements +/// +/// - Must be a tuple struct with exactly one field +/// - The inner type must implement `Bytes` +/// - Supports generic type parameters +/// +/// # Generated Implementation +/// +/// The derive generates a `Bytes` implementation that delegates to the inner type: +/// +/// ```rust,ignore +/// impl Bytes for Wrapper where T: Bytes { +/// type Array = ::Array; +/// +/// fn to_bytes(&self) -> Self::Array { +/// self.0.to_bytes() +/// } +/// fn from_bytes(bytes: &[u8]) -> Result { +/// Ok(Self(::from_bytes(bytes)?)) +/// } +/// } +/// ``` +/// +/// # Example +/// +/// ```rust,ignore +/// use vecdb::{Bytes, BytesVec}; +/// +/// #[derive(Bytes)] +/// struct UserId(u64); +/// +/// #[derive(Bytes)] +/// struct Timestamp(T); // Generic types supported +/// ``` +#[proc_macro_derive(Bytes)] +pub fn derive_bytes(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let struct_name = &input.ident; + let generics = &input.generics; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let inner_type = match &input.data { + Data::Struct(DataStruct { + fields: Fields::Unnamed(fields), + .. + }) if fields.unnamed.len() == 1 => &fields.unnamed[0].ty, + _ => { + return syn::Error::new_spanned( + &input.ident, + "Bytes can only be derived for single-field tuple structs", + ) + .to_compile_error() + .into(); + } + }; + + // Check if we have generic parameters + let has_generics = !generics.params.is_empty(); + + let expanded = if has_generics { + let where_clause = if where_clause.is_some() { + quote! { #where_clause #inner_type: ::vecdb::Bytes, } + } else { + quote! { where #inner_type: ::vecdb::Bytes, } + }; + + quote! { + impl #impl_generics ::vecdb::Bytes for #struct_name #ty_generics #where_clause { + type Array = <#inner_type as ::vecdb::Bytes>::Array; + const IS_NATIVE_LAYOUT: bool = + <#inner_type as ::vecdb::Bytes>::IS_NATIVE_LAYOUT + && ::core::mem::size_of::() == ::core::mem::size_of::<#inner_type>() + && ::core::mem::align_of::() == ::core::mem::align_of::<#inner_type>(); + + fn to_bytes(&self) -> Self::Array { + self.0.to_bytes() + } + + fn from_bytes(bytes: &[u8]) -> ::vecdb::Result { + Ok(Self(<#inner_type>::from_bytes(bytes)?)) + } + } + } + } else { + quote! { + impl ::vecdb::Bytes for #struct_name { + type Array = <#inner_type as ::vecdb::Bytes>::Array; + const IS_NATIVE_LAYOUT: bool = + <#inner_type as ::vecdb::Bytes>::IS_NATIVE_LAYOUT + && ::core::mem::size_of::() == ::core::mem::size_of::<#inner_type>() + && ::core::mem::align_of::() == ::core::mem::align_of::<#inner_type>(); + + fn to_bytes(&self) -> Self::Array { + self.0.to_bytes() + } + + fn from_bytes(bytes: &[u8]) -> ::vecdb::Result { + Ok(Self(<#inner_type>::from_bytes(bytes)?)) + } + } + } + }; + + TokenStream::from(expanded) +} + +/// Derives the `Pco` trait for single-field tuple structs containing numeric types. +/// +/// This macro enables custom wrapper types to work with `PcoVec` for compressed storage +/// of numeric data using Pcodec compression. +/// +/// # Requirements +/// +/// - Must be a tuple struct with exactly one field +/// - The inner type must implement `Pco` (numeric types: u16-u64, i16-i64, f32, f64) +/// - Supports generic type parameters +/// +/// # Generated Implementation +/// +/// The derive generates three trait implementations: +/// +/// 1. `Bytes` - For serialization (same as `#[derive(Bytes)]`) +/// 2. `Pco` - Specifies the numeric type for compression +/// 3. `TransparentPco` - Marker trait for transparent wrappers +/// +/// ```rust,ignore +/// impl Pco for Wrapper where T: Pco + Bytes { +/// type NumberType = ::NumberType; +/// } +/// +/// impl TransparentPco<::NumberType> for Wrapper +/// where T: Pco + Bytes {} +/// +/// impl Bytes for Wrapper where T: Pco + Bytes { +/// // ... same as Bytes derive +/// } +/// ``` +/// +/// The `NumberType` is automatically propagated from the inner type, ensuring the +/// wrapper has the same compression characteristics. +/// +/// # Example +/// +/// ```rust,ignore +/// use vecdb::{Pco, PcoVec}; +/// +/// #[derive(Pco)] +/// struct Price(f64); +/// +/// #[derive(Pco)] +/// struct NumericWrapper(T); // Generic types supported +/// +/// // Nested generics work too +/// #[derive(Pco)] +/// struct Container(NumericWrapper); +/// ``` +#[proc_macro_derive(Pco)] +pub fn derive_pco(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let struct_name = &input.ident; + let generics = &input.generics; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let inner_type = match &input.data { + Data::Struct(DataStruct { + fields: Fields::Unnamed(fields), + .. + }) if fields.unnamed.len() == 1 => &fields.unnamed[0].ty, + _ => { + return syn::Error::new_spanned( + &input.ident, + "Pco can only be derived for single-field tuple structs", + ) + .to_compile_error() + .into(); + } + }; + + // Check if we have generic parameters + let has_generics = !generics.params.is_empty(); + + let expanded = if has_generics { + // For generic types, we need both Pco and Bytes bounds because: + // - Pco trait requires the NumberType + // - We call to_bytes/from_bytes methods which require Bytes + let where_clause = if where_clause.is_some() { + quote! { #where_clause #inner_type: ::vecdb::Pco + ::vecdb::Bytes, } + } else { + quote! { where #inner_type: ::vecdb::Pco + ::vecdb::Bytes, } + }; + + quote! { + impl #impl_generics ::vecdb::Bytes for #struct_name #ty_generics #where_clause { + type Array = <#inner_type as ::vecdb::Bytes>::Array; + const IS_NATIVE_LAYOUT: bool = + <#inner_type as ::vecdb::Bytes>::IS_NATIVE_LAYOUT + && ::core::mem::size_of::() == ::core::mem::size_of::<#inner_type>() + && ::core::mem::align_of::() == ::core::mem::align_of::<#inner_type>(); + + fn to_bytes(&self) -> Self::Array { + self.0.to_bytes() + } + + fn from_bytes(bytes: &[u8]) -> ::vecdb::Result { + Ok(Self(<#inner_type>::from_bytes(bytes)?)) + } + } + + impl #impl_generics ::vecdb::TransparentPco<<#inner_type as ::vecdb::Pco>::NumberType> for #struct_name #ty_generics #where_clause {} + + impl #impl_generics ::vecdb::Pco for #struct_name #ty_generics #where_clause { + type NumberType = <#inner_type as ::vecdb::Pco>::NumberType; + } + } + } else { + quote! { + impl ::vecdb::Bytes for #struct_name { + type Array = <#inner_type as ::vecdb::Bytes>::Array; + const IS_NATIVE_LAYOUT: bool = + <#inner_type as ::vecdb::Bytes>::IS_NATIVE_LAYOUT + && ::core::mem::size_of::() == ::core::mem::size_of::<#inner_type>() + && ::core::mem::align_of::() == ::core::mem::align_of::<#inner_type>(); + + fn to_bytes(&self) -> Self::Array { + self.0.to_bytes() + } + + fn from_bytes(bytes: &[u8]) -> ::vecdb::Result { + Ok(Self(<#inner_type>::from_bytes(bytes)?)) + } + } + + impl ::vecdb::TransparentPco<<#inner_type as ::vecdb::Pco>::NumberType> for #struct_name {} + + impl ::vecdb::Pco for #struct_name { + type NumberType = <#inner_type as ::vecdb::Pco>::NumberType; + } + } + }; + + TokenStream::from(expanded) +} diff --git a/crates/vecdb_derive/tests/bytes.rs b/crates/vecdb_derive/tests/bytes.rs new file mode 100644 index 000000000..7429a6a5d --- /dev/null +++ b/crates/vecdb_derive/tests/bytes.rs @@ -0,0 +1,76 @@ +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, AnyVec, Bytes, BytesVec, Database, HEADER_OFFSET, ImportableVec, ReadableVec, + Version, WritableVec, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct Timestamp(u64); + +#[test] +fn test_derive_bytes_vec_value() -> vecdb::Result<()> { + const { assert!(Timestamp::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: BytesVec = BytesVec::import(&db, "test", Version::TWO)?; + + // Test push + vec.push(Timestamp(12345)); + vec.push(Timestamp(67890)); + vec.push(Timestamp(111213)); + + // Test write + vec.write()?; + + let expected = [ + 12_345_u64.to_le_bytes(), + 67_890_u64.to_le_bytes(), + 111_213_u64.to_le_bytes(), + ] + .concat(); + assert_eq!( + &vec.region().create_reader().read_all()[HEADER_OFFSET..], + expected + ); + + // Test collect + let collected: Vec = vec.collect(); + assert_eq!( + collected, + vec![Timestamp(12345), Timestamp(67890), Timestamp(111213)] + ); + + // Test length + assert_eq!(vec.len(), 3); + + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct Price(f64); + +#[test] +fn test_derive_with_float() -> vecdb::Result<()> { + const { assert!(Price::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: BytesVec = BytesVec::import(&db, "prices", Version::TWO)?; + + vec.push(Price(19.99)); + vec.push(Price(29.99)); + vec.push(Price(39.99)); + + vec.write()?; + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 3); + assert_eq!(collected[0], Price(19.99)); + assert_eq!(collected[1], Price(29.99)); + assert_eq!(collected[2], Price(39.99)); + + Ok(()) +} diff --git a/crates/vecdb_derive/tests/bytes_generics.rs b/crates/vecdb_derive/tests/bytes_generics.rs new file mode 100644 index 000000000..7c35f9246 --- /dev/null +++ b/crates/vecdb_derive/tests/bytes_generics.rs @@ -0,0 +1,122 @@ +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, Bytes, BytesVec, Database, ImportableVec, ReadableVec, Version, WritableVec, +}; + +// Test with a single generic parameter +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct Wrapper(T); + +// Test with nested generics +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct Container(Wrapper); + +// Test with float type +#[derive(Debug, Clone, Copy, PartialEq, Bytes)] +struct FloatWrapper(T); + +#[test] +fn test_derive_bytes_with_single_generic() -> vecdb::Result<()> { + const { assert!(Wrapper::::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: BytesVec> = BytesVec::import(&db, "test_u64", Version::TWO)?; + + vec.push(Wrapper(100)); + vec.push(Wrapper(200)); + vec.push(Wrapper(300)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected.len(), 3); + assert_eq!(collected[0], Wrapper(100)); + assert_eq!(collected[1], Wrapper(200)); + assert_eq!(collected[2], Wrapper(300)); + + Ok(()) +} + +#[test] +fn test_derive_bytes_with_different_types() -> vecdb::Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + // Test with u32 + { + let mut vec: BytesVec> = + BytesVec::import(&db, "test_u32", Version::TWO)?; + + vec.push(Wrapper(42)); + vec.push(Wrapper(84)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected, vec![Wrapper(42), Wrapper(84)]); + } + + // Test with i64 + { + let mut vec: BytesVec> = + BytesVec::import(&db, "test_i64", Version::TWO)?; + + vec.push(Wrapper(-100)); + vec.push(Wrapper(100)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected, vec![Wrapper(-100), Wrapper(100)]); + } + + Ok(()) +} + +// Test with nested generics +#[test] +fn test_derive_bytes_with_nested_generics() -> vecdb::Result<()> { + const { assert!(Container::::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: BytesVec> = + BytesVec::import(&db, "test_nested", Version::TWO)?; + + vec.push(Container(Wrapper(111))); + vec.push(Container(Wrapper(222))); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected.len(), 2); + assert_eq!(collected[0], Container(Wrapper(111))); + assert_eq!(collected[1], Container(Wrapper(222))); + + Ok(()) +} + +// Test with float type +#[test] +fn test_derive_bytes_with_float_generic() -> vecdb::Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: BytesVec> = + BytesVec::import(&db, "test_float", Version::TWO)?; + + vec.push(FloatWrapper(3.144)); + vec.push(FloatWrapper(2.71)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected.len(), 2); + assert_eq!(collected[0], FloatWrapper(3.144)); + assert_eq!(collected[1], FloatWrapper(2.71)); + + Ok(()) +} diff --git a/crates/vecdb_derive/tests/pco.rs b/crates/vecdb_derive/tests/pco.rs new file mode 100644 index 000000000..093fad550 --- /dev/null +++ b/crates/vecdb_derive/tests/pco.rs @@ -0,0 +1,65 @@ +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, AnyVec, Bytes, Database, ImportableVec, Pco, PcoVec, ReadableVec, Version, + WritableVec, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Timestamp(u64); + +#[test] +fn test_derive_pco_vec_value() -> vecdb::Result<()> { + const { assert!(Timestamp::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: PcoVec = PcoVec::import(&db, "test", Version::TWO)?; + + // Test push + vec.push(Timestamp(12345)); + vec.push(Timestamp(67890)); + vec.push(Timestamp(111213)); + + // Test write + vec.write()?; + + // Test collect + let collected: Vec = vec.collect(); + assert_eq!( + collected, + vec![Timestamp(12345), Timestamp(67890), Timestamp(111213)] + ); + + // Test length + assert_eq!(vec.len(), 3); + + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Price(f64); + +#[test] +fn test_derive_with_float() -> vecdb::Result<()> { + const { assert!(Price::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: PcoVec = PcoVec::import(&db, "prices", Version::TWO)?; + + vec.push(Price(19.99)); + vec.push(Price(29.99)); + vec.push(Price(39.99)); + + vec.write()?; + + let collected: Vec = vec.collect(); + assert_eq!(collected.len(), 3); + assert_eq!(collected[0], Price(19.99)); + assert_eq!(collected[1], Price(29.99)); + assert_eq!(collected[2], Price(39.99)); + + Ok(()) +} diff --git a/crates/vecdb_derive/tests/pco_generics.rs b/crates/vecdb_derive/tests/pco_generics.rs new file mode 100644 index 000000000..48bb1b785 --- /dev/null +++ b/crates/vecdb_derive/tests/pco_generics.rs @@ -0,0 +1,139 @@ +use tempfile::TempDir; +use vecdb::{ + AnyStoredVec, Bytes, Database, ImportableVec, Pco, PcoVec, ReadableVec, Version, WritableVec, +}; + +// Test with a single generic parameter +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Wrapper(T); + +// Test with nested generics +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct Container(Wrapper); + +// Test with float type +#[derive(Debug, Clone, Copy, PartialEq, Pco)] +struct FloatWrapper(T); + +#[test] +fn test_derive_pco_with_single_generic() -> vecdb::Result<()> { + const { assert!(Wrapper::::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: PcoVec> = PcoVec::import(&db, "test_u64", Version::TWO)?; + + vec.push(Wrapper(100)); + vec.push(Wrapper(200)); + vec.push(Wrapper(300)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected.len(), 3); + assert_eq!(collected[0], Wrapper(100)); + assert_eq!(collected[1], Wrapper(200)); + assert_eq!(collected[2], Wrapper(300)); + + Ok(()) +} + +#[test] +fn test_derive_pco_with_different_types() -> vecdb::Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + // Test with u32 + { + let mut vec: PcoVec> = PcoVec::import(&db, "test_u32", Version::TWO)?; + + vec.push(Wrapper(42)); + vec.push(Wrapper(84)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected, vec![Wrapper(42), Wrapper(84)]); + } + + // Test with i64 + { + let mut vec: PcoVec> = PcoVec::import(&db, "test_i64", Version::TWO)?; + + vec.push(Wrapper(-100)); + vec.push(Wrapper(100)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected, vec![Wrapper(-100), Wrapper(100)]); + } + + Ok(()) +} + +// Test with nested generics +#[test] +fn test_derive_pco_with_nested_generics() -> vecdb::Result<()> { + const { assert!(Container::::IS_NATIVE_LAYOUT) }; + + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: PcoVec> = PcoVec::import(&db, "test_nested", Version::TWO)?; + + vec.push(Container(Wrapper(111))); + vec.push(Container(Wrapper(222))); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected.len(), 2); + assert_eq!(collected[0], Container(Wrapper(111))); + assert_eq!(collected[1], Container(Wrapper(222))); + + Ok(()) +} + +// Test with float type +#[test] +fn test_derive_pco_with_float_generic() -> vecdb::Result<()> { + let temp = TempDir::new()?; + let db = Database::open(temp.path())?; + + let mut vec: PcoVec> = + PcoVec::import(&db, "test_float", Version::TWO)?; + + vec.push(FloatWrapper(3.144)); + vec.push(FloatWrapper(2.71)); + + vec.write()?; + + let collected: Vec> = vec.collect(); + assert_eq!(collected.len(), 2); + assert_eq!(collected[0], FloatWrapper(3.144)); + assert_eq!(collected[1], FloatWrapper(2.71)); + + Ok(()) +} + +// Test that Pco NumberType is correctly propagated through generics +#[test] +fn test_pco_number_type_with_generic() { + // Verify that the NumberType is correctly set + assert_eq!( + std::any::TypeId::of::< as Pco>::NumberType>(), + std::any::TypeId::of::() + ); + + assert_eq!( + std::any::TypeId::of::< as Pco>::NumberType>(), + std::any::TypeId::of::() + ); + + assert_eq!( + std::any::TypeId::of::< as Pco>::NumberType>(), + std::any::TypeId::of::() + ); +}