mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 17:55:26 +01:00
rust volume: bulk-load needle maps on redb 4.6 and compact the old floor (#12695)
* seaweed-volume: expect a fresh redb file to start at 72 KiB The assertion fails on redb 4.2.0, whose empty database is about 1 MiB. * seaweed-volume: bulk-load redb rebuilds with the stable cursor redb 4.6.0 has no experimental_cursor feature. Full rebuilds insert sorted needle ids through insert_before, and a new .rdb starts at 72 KiB. * seaweed-volume: test which redb files are still at the old floor The gate is missing, so the test does not compile yet. * seaweed-volume: decide when a reused redb is still at the old floor Compact only a file above 72 KiB and at most 1 MiB plus 8 KiB, and only when floor_compact has not been recorded. * seaweed-volume: test one-shot compact of an old-floor redb Reuse, shrink, and storage-error recovery are specified. The helpers they call are not implemented, so the tests do not compile yet. * seaweed-volume: compact a reused redb that is still at the old floor One compact runs when the file is above 72 KiB and at most 1 MiB plus 8 KiB. A storage error drops the handle and opens the same path again so the volume does not keep a PreviousIo latch. * seaweed-volume: publish a redb replay before compacting the old floor compact() is what would make a Durability::None tail durable, and a storage error then drops that tail on reopen. An in-band file is committed first. The floor_compact flag is read only inside the band, so a meta error cannot rebuild a larger index. * seaweed-volume: drop the redb experimental cursor CI step redb 4.6 has no redb-experimental-cursor feature. The needle-map tests that used it now run in the normal cargo test step. * seaweed-volume: drop the experimental-api-5 notes on redb reads redb's experimental-api-5 can no longer be enabled, so these reads always use the inherent ReadOnlyTable::get. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-volume: drop the always-false flag from the old-floor band check Both callers pass false and read floor_compact on their own, so the band check only needs the file length. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
4 files changed
+372
-60
No files matched your search
@@ -168,9 +168,6 @@ jobs:
|
|||||||
- name: Run Rust unit tests
|
- name: Run Rust unit tests
|
||||||
run: cd seaweed-volume && cargo test
|
run: cd seaweed-volume && cargo test
|
||||||
|
|
||||||
- name: Run Rust unit tests (redb experimental cursor)
|
|
||||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
|
||||||
|
|
||||||
rust-unit-tests-windows:
|
rust-unit-tests-windows:
|
||||||
name: Rust Unit Tests (Windows)
|
name: Rust Unit Tests (Windows)
|
||||||
runs-on: windows-latest
|
runs-on: windows-latest
|
||||||
@@ -208,9 +205,6 @@ jobs:
|
|||||||
- name: Run Rust unit tests
|
- name: Run Rust unit tests
|
||||||
run: cd seaweed-volume && cargo test
|
run: cd seaweed-volume && cargo test
|
||||||
|
|
||||||
- name: Run Rust unit tests (redb experimental cursor)
|
|
||||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
|
||||||
|
|
||||||
rust-integration-tests:
|
rust-integration-tests:
|
||||||
name: Rust Integration Tests
|
name: Rust Integration Tests
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-24.04
|
||||||
|
|||||||
Generated
+2
-2
@@ -3195,9 +3195,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redb"
|
name = "redb"
|
||||||
version = "4.2.0"
|
version = "4.6.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36"
|
checksum = "096b1dd5ec053c9c2a5a26a93aafeadf73ebcb710ea54dc3151f17a8e20535ef"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -19,9 +19,6 @@ path = "src/main.rs"
|
|||||||
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
|
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
|
||||||
default = ["5bytes"]
|
default = ["5bytes"]
|
||||||
5bytes = []
|
5bytes = []
|
||||||
# Unstable redb cursor bulk-load for full_rebuild. Off in production.
|
|
||||||
# Pulls redb's experimental_cursor (and therefore experimental-api-5).
|
|
||||||
redb-experimental-cursor = ["redb/experimental_cursor"]
|
|
||||||
|
|
||||||
[lints.clippy]
|
[lints.clippy]
|
||||||
# Protobuf message literals keep `..Default::default()` on purpose: it is
|
# Protobuf message literals keep `..Default::default()` on purpose: it is
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ use std::io::{self, Read, Seek, Write};
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
|
||||||
|
|
||||||
#[cfg(feature = "redb-experimental-cursor")]
|
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
|
|
||||||
mod compact_map;
|
mod compact_map;
|
||||||
@@ -485,6 +484,38 @@ const NEEDLE_TABLE: TableDefinition<u64, &[u8]> = TableDefinition::new("needles"
|
|||||||
const META_TABLE: TableDefinition<&str, u64> = TableDefinition::new("meta");
|
const META_TABLE: TableDefinition<&str, u64> = TableDefinition::new("meta");
|
||||||
const META_IDX_SIZE: &str = "idx_size";
|
const META_IDX_SIZE: &str = "idx_size";
|
||||||
|
|
||||||
|
/// redb 4.6 starts a new file at 72 KiB: 64 KiB of pages plus the header
|
||||||
|
/// and the region tracker.
|
||||||
|
const REDB_SMALL_FILE_BYTES: u64 = 72 * 1024;
|
||||||
|
/// redb 4.2 started at 1 MiB of pages plus that same 8 KiB.
|
||||||
|
const REDB_OLD_FLOOR_BYTES: u64 = 1024 * 1024 + 8 * 1024;
|
||||||
|
/// Meta value `1` means a compact of a file in the old-floor band already finished.
|
||||||
|
const META_FLOOR_COMPACT: &str = "floor_compact";
|
||||||
|
|
||||||
|
/// True when a reused `.rdb` is still inside the old 4.2 starting size.
|
||||||
|
/// A file one byte past it has grown.
|
||||||
|
fn should_compact_old_floor(len: u64) -> bool {
|
||||||
|
len > REDB_SMALL_FILE_BYTES && len <= REDB_OLD_FLOOR_BYTES
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum CompactFollowUp {
|
||||||
|
/// Drop the `Database` and open the same path once. Do not compact again.
|
||||||
|
ReopenSkipCompact,
|
||||||
|
/// The handle can still begin a write. Do not record `floor_compact`.
|
||||||
|
KeepHandle,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compact_follow_up(err: &redb::CompactionError) -> CompactFollowUp {
|
||||||
|
match err {
|
||||||
|
redb::CompactionError::Storage(_) => CompactFollowUp::ReopenSkipCompact,
|
||||||
|
redb::CompactionError::TransactionInProgress
|
||||||
|
| redb::CompactionError::PersistentSavepointExists
|
||||||
|
| redb::CompactionError::EphemeralSavepointExists => CompactFollowUp::KeepHandle,
|
||||||
|
_ => CompactFollowUp::KeepHandle,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Writes between two durable redb checkpoints. Every non-durable commit
|
/// Writes between two durable redb checkpoints. Every non-durable commit
|
||||||
/// leaves an entry in redb's transaction tracker and pins the pages that
|
/// leaves an entry in redb's transaction tracker and pins the pages that
|
||||||
/// were on disk at the last durable commit; only a durable commit clears
|
/// were on disk at the last durable commit; only a durable commit clears
|
||||||
@@ -648,9 +679,6 @@ impl RedbNeedleMap {
|
|||||||
let meta = txn
|
let meta = txn
|
||||||
.open_table(META_TABLE)
|
.open_table(META_TABLE)
|
||||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||||
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
|
|
||||||
// ReadableTable::get guard borrows `meta`; edition 2024 drops the tail
|
|
||||||
// expression's temporaries before `meta`, so no extra binding is needed.
|
|
||||||
match meta.get(META_IDX_SIZE) {
|
match meta.get(META_IDX_SIZE) {
|
||||||
Ok(Some(guard)) => Ok(Some(guard.value())),
|
Ok(Some(guard)) => Ok(Some(guard.value())),
|
||||||
Ok(None) => Ok(None),
|
Ok(None) => Ok(None),
|
||||||
@@ -658,6 +686,118 @@ impl RedbNeedleMap {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rdb_file_len(&self) -> io::Result<u64> {
|
||||||
|
Ok(std::fs::metadata(&self.rdb_path)?.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_floor_compact(&self) -> io::Result<Option<u64>> {
|
||||||
|
let txn = self
|
||||||
|
.db_or_err()?
|
||||||
|
.begin_read()
|
||||||
|
.map_err(|e| io::Error::other(format!("redb begin_read: {}", e)))?;
|
||||||
|
let meta = txn
|
||||||
|
.open_table(META_TABLE)
|
||||||
|
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||||
|
match meta.get(META_FLOOR_COMPACT) {
|
||||||
|
Ok(Some(guard)) => Ok(Some(guard.value())),
|
||||||
|
Ok(None) => Ok(None),
|
||||||
|
Err(e) => Err(io::Error::other(format!("redb get meta: {}", e))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_floor_compact(&self) -> io::Result<()> {
|
||||||
|
let txn = Self::begin_write_no_fsync(self.db_or_err()?)?;
|
||||||
|
{
|
||||||
|
let mut meta = txn
|
||||||
|
.open_table(META_TABLE)
|
||||||
|
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||||
|
meta.insert(META_FLOOR_COMPACT, 1u64)
|
||||||
|
.map_err(|e| io::Error::other(format!("redb insert meta: {}", e)))?;
|
||||||
|
}
|
||||||
|
txn.commit()
|
||||||
|
.map_err(|e| io::Error::other(format!("redb commit meta: {}", e)))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop the exclusive lock and open the same file. Does not compact.
|
||||||
|
fn reopen_database(&mut self) -> io::Result<()> {
|
||||||
|
self.db.take();
|
||||||
|
let db = Database::builder()
|
||||||
|
.set_cache_size(self.cache_bytes)
|
||||||
|
.open(&self.rdb_path)
|
||||||
|
.map_err(|e| io::Error::other(format!("redb reopen after compact: {}", e)))?;
|
||||||
|
self.db = Some(db);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_floor_compact_if_still_in_band(&self) -> io::Result<()> {
|
||||||
|
let len = match self.rdb_file_len() {
|
||||||
|
Ok(len) => len,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("redb floor compact flag skipped, metadata failed: {}", e);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if should_compact_old_floor(len) {
|
||||||
|
self.save_floor_compact()?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compact a reused file that is still the 4.2 starting allocation.
|
||||||
|
///
|
||||||
|
/// Called only from `try_reuse_rdb`, after the replay transaction has
|
||||||
|
/// committed. `new` and `full_rebuild` already create a 72 KiB file.
|
||||||
|
///
|
||||||
|
/// An empty durable commit publishes a `Durability::None` tail before
|
||||||
|
/// `compact`. A storage error inside `compact` would otherwise drop that
|
||||||
|
/// tail when the handle is reopened. `checkpoint` is not used: during
|
||||||
|
/// reuse `idx_file` is `None` and `idx_file_offset` is 0.
|
||||||
|
fn shrink_old_floor(&mut self) -> io::Result<()> {
|
||||||
|
let len = match self.rdb_file_len() {
|
||||||
|
Ok(len) => len,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("redb floor compact skipped, metadata failed: {}", e);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// The flag is irrelevant outside the band. A meta read error there
|
||||||
|
// must not make `load_from_idx` rebuild a larger index.
|
||||||
|
if !should_compact_old_floor(len) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if self.read_floor_compact()? == Some(1) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// Failure returns Err so `load_from_idx` full-rebuilds. Do not reopen
|
||||||
|
// over a tail that this commit did not publish.
|
||||||
|
let mut txn = self
|
||||||
|
.db_or_err()?
|
||||||
|
.begin_write()
|
||||||
|
.map_err(|e| io::Error::other(format!("redb begin_write: {}", e)))?;
|
||||||
|
txn.set_quick_repair(true);
|
||||||
|
txn.commit()
|
||||||
|
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
||||||
|
let compact_result = self
|
||||||
|
.db
|
||||||
|
.as_mut()
|
||||||
|
.ok_or_else(|| io::Error::other("redb database is closed"))?
|
||||||
|
.compact();
|
||||||
|
match compact_result {
|
||||||
|
Ok(_) => self.record_floor_compact_if_still_in_band(),
|
||||||
|
Err(e) => match compact_follow_up(&e) {
|
||||||
|
CompactFollowUp::ReopenSkipCompact => {
|
||||||
|
tracing::warn!("redb floor compact storage error, reopening: {}", e);
|
||||||
|
self.reopen_database()
|
||||||
|
}
|
||||||
|
CompactFollowUp::KeepHandle => {
|
||||||
|
tracing::warn!("redb floor compact skipped: {}", e);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Test-only read of META `idx_size` through the live handle. See
|
/// Test-only read of META `idx_size` through the live handle. See
|
||||||
/// [`test_support::live_meta_idx_size`] for why durability tests use
|
/// [`test_support::live_meta_idx_size`] for why durability tests use
|
||||||
/// this instead of copying the open `.rdb`.
|
/// this instead of copying the open `.rdb`.
|
||||||
@@ -666,6 +806,11 @@ impl RedbNeedleMap {
|
|||||||
self.read_idx_size_meta().unwrap()
|
self.read_idx_size_meta().unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn live_floor_compact(&self) -> Option<u64> {
|
||||||
|
self.read_floor_compact().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
/// Load from an .idx file, reusing an existing .rdb if it is consistent.
|
/// Load from an .idx file, reusing an existing .rdb if it is consistent.
|
||||||
///
|
///
|
||||||
/// Strategy:
|
/// Strategy:
|
||||||
@@ -774,6 +919,7 @@ impl RedbNeedleMap {
|
|||||||
nm.save_idx_size_meta(idx_size)?;
|
nm.save_idx_size_meta(idx_size)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
nm.shrink_old_floor()?;
|
||||||
Ok(nm)
|
Ok(nm)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -796,8 +942,8 @@ impl RedbNeedleMap {
|
|||||||
/// `Database::create` opens an existing file (`truncate(false)`). Unlink
|
/// `Database::create` opens an existing file (`truncate(false)`). Unlink
|
||||||
/// is best-effort: if it fails (sticky bit, foreign owner) but the path is
|
/// is best-effort: if it fails (sticky bit, foreign owner) but the path is
|
||||||
/// still writable, clear the leftover needles table before inserting.
|
/// still writable, clear the leftover needles table before inserting.
|
||||||
/// Live keys come from a `BTreeMap` so inserts are in needle-id order and
|
/// Live keys come from a `BTreeMap` so `insert_before` walks them in
|
||||||
/// redb 4.2.0 packs leaves, without a second Vec + sort scratch.
|
/// needle-id order and packs leaves, without a second Vec + sort scratch.
|
||||||
fn full_rebuild<R: Read + Seek>(
|
fn full_rebuild<R: Read + Seek>(
|
||||||
db_path: &str,
|
db_path: &str,
|
||||||
reader: &mut R,
|
reader: &mut R,
|
||||||
@@ -849,43 +995,26 @@ impl RedbNeedleMap {
|
|||||||
.map_err(|e| io::Error::other(format!("redb retain: {}", e)))?;
|
.map_err(|e| io::Error::other(format!("redb retain: {}", e)))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "redb-experimental-cursor"))]
|
let mut cursor = table
|
||||||
{
|
.upper_bound_mut(Bound::<u64>::Unbounded)
|
||||||
for (key, nv) in &entries {
|
.map_err(|e| {
|
||||||
let key_u64: u64 = (*key).into();
|
io::Error::new(io::ErrorKind::Other, format!("redb upper_bound_mut: {}", e))
|
||||||
let packed = pack_needle_value(nv);
|
|
||||||
table
|
|
||||||
.insert(key_u64, packed.as_slice())
|
|
||||||
.map_err(|e| io::Error::other(format!("redb insert: {}", e)))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(feature = "redb-experimental-cursor")]
|
|
||||||
{
|
|
||||||
let mut cursor =
|
|
||||||
table
|
|
||||||
.upper_bound_mut(Bound::<u64>::Unbounded)
|
|
||||||
.map_err(|e| {
|
|
||||||
io::Error::new(
|
|
||||||
io::ErrorKind::Other,
|
|
||||||
format!("redb upper_bound_mut: {}", e),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
for (key, nv) in &entries {
|
|
||||||
let key_u64: u64 = (*key).into();
|
|
||||||
let packed = pack_needle_value(nv);
|
|
||||||
cursor
|
|
||||||
.insert_before(key_u64, packed.as_slice())
|
|
||||||
.map_err(|e| {
|
|
||||||
io::Error::new(
|
|
||||||
io::ErrorKind::Other,
|
|
||||||
format!("redb insert_before: {}", e),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
cursor.close().map_err(|e| {
|
|
||||||
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
|
|
||||||
})?;
|
})?;
|
||||||
|
for (key, nv) in &entries {
|
||||||
|
let key_u64: u64 = (*key).into();
|
||||||
|
let packed = pack_needle_value(nv);
|
||||||
|
cursor
|
||||||
|
.insert_before(key_u64, packed.as_slice())
|
||||||
|
.map_err(|e| {
|
||||||
|
io::Error::new(
|
||||||
|
io::ErrorKind::Other,
|
||||||
|
format!("redb insert_before: {}", e),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
}
|
}
|
||||||
|
cursor.close().map_err(|e| {
|
||||||
|
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
|
||||||
|
})?;
|
||||||
}
|
}
|
||||||
txn.commit()
|
txn.commit()
|
||||||
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
||||||
@@ -1048,9 +1177,6 @@ impl RedbNeedleMap {
|
|||||||
let table = txn
|
let table = txn
|
||||||
.open_table(NEEDLE_TABLE)
|
.open_table(NEEDLE_TABLE)
|
||||||
.map_err(|e| io::Error::other(format!("redb open_table: {}", e)))?;
|
.map_err(|e| io::Error::other(format!("redb open_table: {}", e)))?;
|
||||||
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
|
|
||||||
// ReadableTable::get guard borrows `table`; edition 2024 drops the tail
|
|
||||||
// expression's temporaries before `table`, so no extra binding is needed.
|
|
||||||
match table.get(key_u64) {
|
match table.get(key_u64) {
|
||||||
Ok(Some(guard)) => Ok(packed_to_needle_value(guard.value())),
|
Ok(Some(guard)) => Ok(packed_to_needle_value(guard.value())),
|
||||||
Ok(None) => Ok(None),
|
Ok(None) => Ok(None),
|
||||||
@@ -1181,7 +1307,7 @@ impl RedbNeedleMap {
|
|||||||
/// Reopen the database from `.idx` after a non-`TransactionPoisoned`
|
/// Reopen the database from `.idx` after a non-`TransactionPoisoned`
|
||||||
/// commit error.
|
/// commit error.
|
||||||
///
|
///
|
||||||
/// redb 4.2.0 can make a `Durability::None` commit visible *before*
|
/// redb can make a `Durability::None` commit visible *before*
|
||||||
/// returning `Err(CommitError::Storage(..))`. In that state the database
|
/// returning `Err(CommitError::Storage(..))`. In that state the database
|
||||||
/// refuses further write transactions, so the map cannot be used as-is.
|
/// refuses further write transactions, so the map cannot be used as-is.
|
||||||
/// The `.idx` row that triggered the commit is **not** truncated — the
|
/// The `.idx` row that triggered the commit is **not** truncated — the
|
||||||
@@ -1603,9 +1729,10 @@ pub(crate) mod test_support {
|
|||||||
/// every `put`/`delete` also commits non-durably, so before the first
|
/// every `put`/`delete` also commits non-durably, so before the first
|
||||||
/// checkpoint this is the load-time value (`Some(0)` for a fresh map) —
|
/// checkpoint this is the load-time value (`Some(0)` for a fresh map) —
|
||||||
/// NOT the crash-durable `None` a copy of the open `.rdb` would show.
|
/// NOT the crash-durable `None` a copy of the open `.rdb` would show.
|
||||||
/// A live read is the only portable observation: redb 4.2.0 takes an
|
/// A live read is the only portable observation: redb's default
|
||||||
/// exclusive whole-file lock, which is advisory on Unix but mandatory
|
/// `ExclusiveWriter` takes an exclusive whole-file lock, which is
|
||||||
/// on Windows, so copying the open `.rdb` fails there with OS error 33.
|
/// advisory on Unix but mandatory on Windows, so copying the open
|
||||||
|
/// `.rdb` fails there with OS error 33.
|
||||||
///
|
///
|
||||||
/// It still pins the property under test: the only *durable* META
|
/// It still pins the property under test: the only *durable* META
|
||||||
/// writer is `checkpoint`, so any value other than the load-time one
|
/// writer is `checkpoint`, so any value other than the load-time one
|
||||||
@@ -1763,6 +1890,41 @@ mod tests {
|
|||||||
NeedleMapKind::Redb.redb_cache_bytes()
|
NeedleMapKind::Redb.redb_cache_bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_redb_new_file_is_at_most_72kib() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db_path = dir.path().join("fresh.rdb");
|
||||||
|
let nm = RedbNeedleMap::new(db_path.to_str().unwrap(), redb_test_cache()).unwrap();
|
||||||
|
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
assert!(
|
||||||
|
len <= REDB_SMALL_FILE_BYTES,
|
||||||
|
"fresh redb file is {len} bytes, above the 72 KiB start"
|
||||||
|
);
|
||||||
|
drop(nm);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_should_compact_old_floor_bounds() {
|
||||||
|
let small = REDB_SMALL_FILE_BYTES;
|
||||||
|
let floor = REDB_OLD_FLOOR_BYTES;
|
||||||
|
assert!(!should_compact_old_floor(small));
|
||||||
|
assert!(should_compact_old_floor(small + 1));
|
||||||
|
assert!(should_compact_old_floor(floor));
|
||||||
|
assert!(!should_compact_old_floor(floor + 1));
|
||||||
|
assert!(!should_compact_old_floor(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reload_redb(db_path: &std::path::Path, idx_path: &std::path::Path) -> RedbNeedleMap {
|
||||||
|
let mut idx = std::fs::File::open(idx_path).unwrap();
|
||||||
|
RedbNeedleMap::load_from_idx(
|
||||||
|
db_path.to_str().unwrap(),
|
||||||
|
&mut idx,
|
||||||
|
Version::current(),
|
||||||
|
redb_test_cache(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
/// Open a redb map on an empty .idx with an append writer attached, the
|
/// Open a redb map on an empty .idx with an append writer attached, the
|
||||||
/// way `Volume::load_index_redb` opens a writable volume.
|
/// way `Volume::load_index_redb` opens a writable volume.
|
||||||
fn open_writable_redb(
|
fn open_writable_redb(
|
||||||
@@ -1895,7 +2057,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "redb-experimental-cursor")]
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_redb_full_rebuild_insert_before_reads_back_thousands_of_shuffled_keys() {
|
fn test_redb_full_rebuild_insert_before_reads_back_thousands_of_shuffled_keys() {
|
||||||
// Enough keys to split leaves. insert_before vs table.insert only
|
// Enough keys to split leaves. insert_before vs table.insert only
|
||||||
@@ -2474,4 +2635,164 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_redb_fresh_reuse_does_not_record_floor_compact() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||||
|
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
assert!(len <= REDB_SMALL_FILE_BYTES, "fresh file is {len} bytes");
|
||||||
|
nm.close();
|
||||||
|
drop(nm);
|
||||||
|
let loaded = reload_redb(&db_path, &idx_path);
|
||||||
|
assert_eq!(loaded.live_floor_compact(), None);
|
||||||
|
assert!(loaded.get(NeedleId(1)).unwrap().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_redb_in_band_file_is_compacted_once() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||||
|
let mut n = 0u64;
|
||||||
|
loop {
|
||||||
|
n += 1;
|
||||||
|
nm.put(
|
||||||
|
NeedleId(n),
|
||||||
|
Offset::from_actual_offset((n as i64).saturating_mul(8)),
|
||||||
|
Size(8),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
if should_compact_old_floor(len) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
n < 2_000_000,
|
||||||
|
"file grew to {len} bytes without entering the old-floor band"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
nm.checkpoint(true).unwrap();
|
||||||
|
nm.close();
|
||||||
|
drop(nm);
|
||||||
|
|
||||||
|
let mut loaded = reload_redb(&db_path, &idx_path);
|
||||||
|
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
let flag = loaded.live_floor_compact();
|
||||||
|
if should_compact_old_floor(len) {
|
||||||
|
assert_eq!(flag, Some(1));
|
||||||
|
} else {
|
||||||
|
assert!(len <= REDB_SMALL_FILE_BYTES, "compacted file is {len} bytes");
|
||||||
|
assert_eq!(flag, None);
|
||||||
|
}
|
||||||
|
let first = loaded.get(NeedleId(1)).unwrap();
|
||||||
|
assert!(first.is_some());
|
||||||
|
let last = loaded.get(NeedleId(n)).unwrap();
|
||||||
|
assert!(last.is_some());
|
||||||
|
// close checkpoints the non-durable floor_compact flag, then drop
|
||||||
|
// releases the exclusive lock before the second open.
|
||||||
|
loaded.close();
|
||||||
|
drop(loaded);
|
||||||
|
|
||||||
|
let again = reload_redb(&db_path, &idx_path);
|
||||||
|
assert_eq!(std::fs::metadata(&db_path).unwrap().len(), len);
|
||||||
|
assert_eq!(again.live_floor_compact(), flag);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_redb_sparse_old_floor_shrinks_on_reuse() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||||
|
let mut n = 0u64;
|
||||||
|
loop {
|
||||||
|
n += 1;
|
||||||
|
nm.put(
|
||||||
|
NeedleId(n),
|
||||||
|
Offset::from_actual_offset((n as i64).saturating_mul(8)),
|
||||||
|
Size(8),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
if should_compact_old_floor(len) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
assert!(n < 2_000_000, "file is {len} bytes and never entered the band");
|
||||||
|
}
|
||||||
|
assert!(n > 32, "band was reached with only {n} keys");
|
||||||
|
nm.checkpoint(true).unwrap();
|
||||||
|
nm.close();
|
||||||
|
drop(nm);
|
||||||
|
|
||||||
|
let keep = 32u64;
|
||||||
|
let idx = std::fs::OpenOptions::new().write(true).open(&idx_path).unwrap();
|
||||||
|
idx.set_len(keep * NEEDLE_MAP_ENTRY_SIZE as u64).unwrap();
|
||||||
|
drop(idx);
|
||||||
|
|
||||||
|
let db = redb::Database::builder()
|
||||||
|
.set_cache_size(redb_test_cache())
|
||||||
|
.open(db_path.to_str().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let txn = db.begin_write().unwrap();
|
||||||
|
{
|
||||||
|
let mut table = txn.open_table(NEEDLE_TABLE).unwrap();
|
||||||
|
for key in (keep + 1)..=n {
|
||||||
|
table.remove(key).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let mut meta = txn.open_table(META_TABLE).unwrap();
|
||||||
|
meta.insert(META_IDX_SIZE, keep * NEEDLE_MAP_ENTRY_SIZE as u64)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
txn.commit().unwrap();
|
||||||
|
drop(db);
|
||||||
|
|
||||||
|
let before = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
let loaded = reload_redb(&db_path, &idx_path);
|
||||||
|
let after = std::fs::metadata(&db_path).unwrap().len();
|
||||||
|
assert!(after < before, "reuse left the file at {after} bytes, was {before}");
|
||||||
|
assert!(
|
||||||
|
after <= REDB_SMALL_FILE_BYTES,
|
||||||
|
"32 live keys still occupy {after} bytes"
|
||||||
|
);
|
||||||
|
assert!(loaded.get(NeedleId(1)).unwrap().is_some());
|
||||||
|
assert!(loaded.get(NeedleId(keep)).unwrap().is_some());
|
||||||
|
assert!(loaded.get(NeedleId(keep + 1)).unwrap().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compact_follow_up_reopens_only_on_storage_error() {
|
||||||
|
use redb::{CompactionError, StorageError};
|
||||||
|
let storage = CompactionError::Storage(StorageError::Io(std::io::Error::other("sync")));
|
||||||
|
assert_eq!(
|
||||||
|
compact_follow_up(&storage),
|
||||||
|
CompactFollowUp::ReopenSkipCompact
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compact_follow_up(&CompactionError::TransactionInProgress),
|
||||||
|
CompactFollowUp::KeepHandle
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compact_follow_up(&CompactionError::PersistentSavepointExists),
|
||||||
|
CompactFollowUp::KeepHandle
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compact_follow_up(&CompactionError::EphemeralSavepointExists),
|
||||||
|
CompactFollowUp::KeepHandle
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_redb_reopen_database_reads_committed_key() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (mut nm, _db_path, _idx_path) = open_writable_redb(dir.path());
|
||||||
|
nm.put(NeedleId(7), Offset::from_actual_offset(64), Size(9))
|
||||||
|
.unwrap();
|
||||||
|
nm.reopen_database().unwrap();
|
||||||
|
let nv = nm.get(NeedleId(7)).unwrap().unwrap();
|
||||||
|
assert_eq!(nv.size, Size(9));
|
||||||
|
nm.put(NeedleId(8), Offset::from_actual_offset(128), Size(10))
|
||||||
|
.unwrap();
|
||||||
|
let nv = nm.get(NeedleId(8)).unwrap().unwrap();
|
||||||
|
assert_eq!(nv.size, Size(10));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user