From aceb089bc8ae9f1463d34e58a12276e695fbf17a Mon Sep 17 00:00:00 2001 From: MindAeon <30555587+mindaeon@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:24:28 +0300 Subject: [PATCH] lance: per-table cleanup retention and on-demand purge of deleted rows (#12739) * lance: per-table cleanup retention and on-demand purge of deleted rows A vector deleted from an index stayed in storage until compaction rewrote its fragment (only above 10% deletions) and the cleanup job's 168 h retention, set once per worker, aged out every version that referenced it. Let a table override retain_hours and min_versions in its Lance config (seaweedfs.cleanup.retain_hours, seaweedfs.cleanup.min_versions), and let it request a purge (seaweedfs.purge.requested). The cleanup job proposes a table with a pending purge regardless of the version floor and then rewrites every fragment with deletions, retrains the indices (a remapped IVF_FLAT index still holds the deleted vector), keeps only the latest version, and records the request as served. A version pinned by a tag fails the purge and leaves the request pending. The test scans every file of a local table for a deleted vector's key, metadata and values, and finds none after the purge. * lance: refuse to purge a sharded vector index A retrain builds every index name over the whole table, so the purge turned shards of one vector index (one name per fragment set) into names that each covered every fragment, indexing every row once per shard. Check before compacting and fail with the request left pending, and retrain only the vector indices. Co-Authored-By: Claude Opus 5.5 * lance: rebuild scalar indices in a purge Compaction only remaps scalar indices, and a retrain leaves them alone, so a purge reported success while a bitmap index still listed the deleted row's value, and under stable row ids a BTree index still held the deleted key. Build each scalar index again from the remaining rows with the parameters it was built with. Co-Authored-By: Claude Opus 5.5 * lance: purge through the cleanup job's retention policy The purge built its own cleanup policy, so a deployment that has the cleanup job keep more than the floor and the window (versions an unleased reader may still be on) would have lost them to a purge. Clean up through retention_policy, the one the job itself uses, with the latest version kept. Mark the request served only once the oldest version left is one the rebuild wrote, and record the rebuild so a purge that has to wait for its cleanup (a tag, or that policy) retries only the cleanup instead of compacting and retraining the indices on every pass. Co-Authored-By: Claude Opus 5.5 * vector: lease the version every list page reads A list page read its version without a lease unless it was a segment that read data and metadata twice. A purge removes every version but the latest, and only a lease tag stops it, so a listing that opened the version before the purge's compaction could lose its files mid-read. Pin it as get_vectors and query_vectors do. Co-Authored-By: Claude Opus 5.5 * lance: give the cleanup job a build's execution time A purge compacts the table and retrains its indices inside a cleanup job, but the job declared no runtime defaults, so the admin held it to its 90 s default and could expire a purge mid-rebuild. Declare the same budget the indices job uses for a retrain. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- seaweed-vector/crates/server/src/service.rs | 9 +- .../crates/lance/src/jobs/cleanup.rs | 193 +++++++-- seaweed-worker/crates/lance/src/jobs/mod.rs | 1 + seaweed-worker/crates/lance/src/jobs/purge.rs | 328 +++++++++++++++ seaweed-worker/crates/lance/tests/purge.rs | 394 ++++++++++++++++++ 5 files changed, 888 insertions(+), 37 deletions(-) create mode 100644 seaweed-worker/crates/lance/src/jobs/purge.rs create mode 100644 seaweed-worker/crates/lance/tests/purge.rs diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index e35e830620..c1fc7395ef 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -908,14 +908,9 @@ impl VectorService for VectorServer { 0 => DEFAULT_PAGE, n => n.min(MAX_PAGE), }; + // A purge keeps only the latest version, so lease the one this page reads. + let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?; if segments > 1 { - // Data and metadata come from a second read of this version, so - // lease it as get_vectors does. - let _lease = if columns.len() > 1 { - self.leases.try_pin(&dataset).await.map_err(internal)? - } else { - None - }; let (vectors, next) = list_segment( &dataset, &columns, diff --git a/seaweed-worker/crates/lance/src/jobs/cleanup.rs b/seaweed-worker/crates/lance/src/jobs/cleanup.rs index 83a992736a..77d1666de6 100644 --- a/seaweed-worker/crates/lance/src/jobs/cleanup.rs +++ b/seaweed-worker/crates/lance/src/jobs/cleanup.rs @@ -3,11 +3,12 @@ use std::collections::HashMap; use anyhow::{Context, Result, anyhow}; use async_trait::async_trait; use chrono::{Duration, Utc}; +use lance::Dataset; use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions}; use seaweed_worker_core::config_form::{form, int_or, int_value, number_field}; use seaweed_worker_core::pb::{ - ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted, - JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor, + AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, + JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor, RunDetectionRequest, }; use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; @@ -15,6 +16,7 @@ use tracing::warn; use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; +use crate::jobs::purge::{self, Retention}; use crate::jobs::{clamp, string_list, table_id}; pub const JOB_TYPE: &str = "lance_cleanup_versions"; @@ -28,7 +30,9 @@ const MAX_MIN_VERSIONS: i64 = 1000; /// Lance keeps every version until something removes it. Lance can also do this /// itself through auto-cleanup, so this job is for deployments that would rather -/// the cluster owned the policy than each writer. +/// the cluster owned the policy than each writer. A table may override the +/// retention in its own config, and may ask for a purge (see [`purge`]), which +/// this job carries out ahead of the retention window. pub struct CleanupVersionsHandler { namespace_url: String, storage: dataset::Storage, @@ -68,9 +72,8 @@ impl CleanupVersionsHandler { /// The oldest version that may be removed while still leaving `min_versions` /// behind, or None when the table has no more than the floor. -async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result> { - let mut versions: Vec = table - .dataset +async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result> { + let mut versions: Vec = dataset .versions() .await? .iter() @@ -83,6 +86,33 @@ async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Resul Ok(Some(versions[versions.len() - min_versions])) } +/// The cleanup that keeps `min_versions` and every version younger than +/// `retain_hours`, or None when the floor leaves nothing to remove. A purge +/// cleans up through it too. +pub async fn retention_policy( + dataset: &Dataset, + retain_hours: i64, + min_versions: usize, +) -> Result> { + let Some(floor) = version_floor(dataset, min_versions).await? else { + return Ok(None); + }; + Ok(Some(CleanupPolicy { + before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)), + // should_clean() ANDs its clauses, so a version has to be both older + // than the window and below the floor to go. + before_version: Some(floor), + // Files this dataset cannot account for are left alone: they may + // belong to a writer that has not committed yet, and deleting them + // would corrupt a commit in flight. + delete_unverified: false, + // A tagged version is pinned on purpose, so refuse rather than + // silently dropping what someone named. + error_if_tagged_old_versions: true, + ..Default::default() + })) +} + #[async_trait] impl JobHandler for CleanupVersionsHandler { fn capability(&self) -> JobTypeCapability { @@ -134,6 +164,12 @@ impl JobHandler for CleanupVersionsHandler { defaults.clone(), )), worker_default_values: defaults, + // A purge compacts and retrains, so it gets a build's time. + admin_runtime_defaults: Some(AdminRuntimeDefaults { + job_type_max_runtime_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS, + execution_timeout_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS, + ..Default::default() + }), ..Default::default() } } @@ -181,10 +217,23 @@ impl JobHandler for CleanupVersionsHandler { continue; } }; + let config = table.dataset.config(); + let pending_purge = purge::pending(config); + let floor = Retention::for_table( + config, + Retention { + retain_hours: DEFAULT_RETAIN_HOURS, + min_versions, + }, + MAX_RETAIN_HOURS, + MAX_MIN_VERSIONS, + ) + .min_versions; // Age is decided at execution against the retention window; a table // at or under the floor cannot lose a version whatever its age, so - // proposing one would only produce a job with nothing to do. - if stats.total_versions <= min_versions { + // proposing one would only produce a job with nothing to do. A + // requested purge ignores the floor: it keeps only the latest. + if pending_purge.is_none() && stats.total_versions <= floor { continue; } let mut parameters: HashMap = HashMap::new(); @@ -193,12 +242,21 @@ impl JobHandler for CleanupVersionsHandler { proposal_id: format!("{JOB_TYPE}:{encoded}"), dedupe_key: format!("{JOB_TYPE}:{encoded}"), job_type: JOB_TYPE.to_string(), - summary: format!("Clean up {encoded} ({} versions)", stats.total_versions), - detail: format!( - "{} versions retained, above the {min_versions} floor; \ - those outside the retention window can go", - stats.total_versions - ), + summary: match &pending_purge { + Some(_) => format!("Purge {encoded} ({} versions)", stats.total_versions), + None => format!("Clean up {encoded} ({} versions)", stats.total_versions), + }, + detail: match &pending_purge { + Some(token) => format!( + "purge requested at {token}: rewrite fragments with deletions \ + and keep only the latest version" + ), + None => format!( + "{} versions retained, above the {floor} floor; \ + those outside the retention window can go", + stats.total_versions + ), + }, parameters, ..Default::default() }); @@ -254,14 +312,33 @@ impl JobHandler for CleanupVersionsHandler { ) as usize; let client = self.client(); - let table = dataset::open(&client, &id, &self.storage).await?; + let mut table = dataset::open(&client, &id, &self.storage).await?; let before = table.stats().await?; + if let Some(token) = purge::pending(table.dataset.config()) { + return self + .execute_purge(request, sender, &job.job_id, &mut table, &token) + .await; + } + let Retention { + retain_hours, + min_versions, + } = Retention::for_table( + table.dataset.config(), + Retention { + retain_hours, + min_versions, + }, + MAX_RETAIN_HOURS, + MAX_MIN_VERSIONS, + ); + // The floor is a promise about how much history survives, so it has to // be applied here and not only when the job was proposed: by the time it // runs, versions may have aged past the retention window, and age alone // would take the table below what the operator asked to keep. - let Some(floor) = version_floor(&table, min_versions).await? else { + let Some(policy) = retention_policy(&table.dataset, retain_hours, min_versions).await? + else { sender.send_completed(JobCompleted { request_id: request.request_id.clone(), job_id: job.job_id.clone(), @@ -288,20 +365,6 @@ impl JobHandler for CleanupVersionsHandler { ..Default::default() })?; - let policy = CleanupPolicy { - before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)), - // should_clean() ANDs its clauses, so a version has to be both older - // than the window and below the floor to go. - before_version: Some(floor), - // Files this dataset cannot account for are left alone: they may - // belong to a writer that has not committed yet, and deleting them - // would corrupt a commit in flight. - delete_unverified: false, - // A tagged version is pinned on purpose, so refuse rather than - // silently dropping what someone named. - error_if_tagged_old_versions: true, - ..Default::default() - }; let stats = cleanup_old_versions(&table.dataset, policy) .await .with_context(|| format!("clean up versions of {}", table.location))?; @@ -343,3 +406,73 @@ impl JobHandler for CleanupVersionsHandler { Ok(()) } } + +impl CleanupVersionsHandler { + async fn execute_purge( + &self, + request: &ExecuteJobRequest, + sender: &dyn ExecutionSender, + job_id: &str, + table: &mut dataset::OpenTable, + token: &str, + ) -> Result<()> { + let stats = purge::purge(&mut table.dataset, token) + .await + .with_context(|| format!("purge {}", table.location))?; + if let Some(counters) = &self.metrics { + counters.versions_removed.inc_by(stats.old_versions); + counters.bytes_reclaimed.inc_by(stats.bytes_removed); + } + let mut output: HashMap = HashMap::new(); + output.insert( + "fragments_rewritten".to_string(), + int_value(stats.fragments_rewritten as i64), + ); + output.insert( + "old_versions_removed".to_string(), + int_value(stats.old_versions as i64), + ); + output.insert( + "bytes_removed".to_string(), + int_value(stats.bytes_removed as i64), + ); + sender.send_completed(JobCompleted { + request_id: request.request_id.clone(), + job_id: job_id.to_string(), + job_type: JOB_TYPE.to_string(), + success: true, + result: Some(JobResult { + output_values: output, + summary: if stats.served { + format!( + "purged: rewrote {} fragments, removed {} versions and {} bytes", + stats.fragments_rewritten, stats.old_versions, stats.bytes_removed + ) + } else { + format!( + "purge pending: rewrote {} fragments, removed {} versions and {} bytes; \ + versions from before the rebuild are still retained", + stats.fragments_rewritten, stats.old_versions, stats.bytes_removed + ) + }, + ..Default::default() + }), + ..Default::default() + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_purge_gets_a_builds_execution_timeout() { + let descriptor = CleanupVersionsHandler::new(String::new()).descriptor(); + let defaults = descriptor.admin_runtime_defaults.unwrap(); + assert_eq!(defaults.execution_timeout_seconds, 3600); + assert_eq!(defaults.job_type_max_runtime_seconds, 3600); + assert!(!defaults.enabled); + } +} diff --git a/seaweed-worker/crates/lance/src/jobs/mod.rs b/seaweed-worker/crates/lance/src/jobs/mod.rs index f654a98bab..54bf5b51d5 100644 --- a/seaweed-worker/crates/lance/src/jobs/mod.rs +++ b/seaweed-worker/crates/lance/src/jobs/mod.rs @@ -9,6 +9,7 @@ pub mod cleanup; pub mod compact; pub mod indices; +pub mod purge; pub mod sort; pub mod vector_index; diff --git a/seaweed-worker/crates/lance/src/jobs/purge.rs b/seaweed-worker/crates/lance/src/jobs/purge.rs new file mode 100644 index 0000000000..25b13b047c --- /dev/null +++ b/seaweed-worker/crates/lance/src/jobs/purge.rs @@ -0,0 +1,328 @@ +//! Per-table retention for version cleanup, and purges: compaction that +//! rewrites every fragment with deletions followed by a cleanup that keeps +//! only the latest version, so the bytes of deleted rows leave storage now +//! rather than after the retention window. +//! +//! Both are set in the table's own Lance config, so any client that can write +//! the table can set them and they travel with it: +//! +//! - `seaweedfs.cleanup.retain_hours` and `seaweedfs.cleanup.min_versions` +//! override the worker's retention for this table. +//! - `seaweedfs.purge.requested` holds a request token ([`request_purge`] +//! writes the current time); a purge is pending while it differs from +//! `seaweedfs.purge.completed`, which the purge sets to the token it served. +//! `seaweedfs.purge.rebuilt` records the token whose rebuild is done. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use anyhow::{Context, Result, bail}; +use chrono::Utc; +use lance::Dataset; +use lance::dataset::cleanup::{RemovalStats, cleanup_old_versions}; +use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::optimize::OptimizeOptions; +use lance_index::{IndexType, is_system_index}; +use lance_table::format::IndexMetadata; +use tracing::warn; + +use crate::jobs::cleanup::retention_policy; + +pub const RETAIN_HOURS_KEY: &str = "seaweedfs.cleanup.retain_hours"; +pub const MIN_VERSIONS_KEY: &str = "seaweedfs.cleanup.min_versions"; +pub const PURGE_REQUESTED_KEY: &str = "seaweedfs.purge.requested"; +pub const PURGE_COMPLETED_KEY: &str = "seaweedfs.purge.completed"; +pub const PURGE_REBUILT_KEY: &str = "seaweedfs.purge.rebuilt"; + +/// The retention cleanup applies to one table: the worker's, unless the +/// table's config overrides a value with one inside the worker's range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Retention { + pub retain_hours: i64, + pub min_versions: usize, +} + +impl Retention { + pub fn for_table( + config: &HashMap, + worker: Retention, + max_retain_hours: i64, + max_min_versions: i64, + ) -> Retention { + let read = |key: &str, min: i64, max: i64| -> Option { + let raw = config.get(key)?; + match raw.trim().parse::() { + Ok(value) if (min..=max).contains(&value) => Some(value), + _ => { + warn!("ignoring table config {key}={raw:?}: not an integer in {min}..={max}"); + None + } + } + }; + Retention { + retain_hours: read(RETAIN_HOURS_KEY, 0, max_retain_hours) + .unwrap_or(worker.retain_hours), + min_versions: read(MIN_VERSIONS_KEY, 1, max_min_versions) + .map(|v| v as usize) + .unwrap_or(worker.min_versions), + } + } +} + +/// The request token of a purge the table is waiting for, if any. +pub fn pending(config: &HashMap) -> Option { + let requested = config.get(PURGE_REQUESTED_KEY).map(|s| s.trim())?; + if requested.is_empty() { + return None; + } + match config.get(PURGE_COMPLETED_KEY) { + Some(completed) if completed.trim() == requested => None, + _ => Some(requested.to_string()), + } +} + +/// Records a purge request on the table and returns its token. The next +/// cleanup detection proposes the purge. +pub async fn request_purge(dataset: &mut Dataset) -> Result { + let token = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + dataset + .update_config([(PURGE_REQUESTED_KEY, token.as_str())]) + .await + .context("record purge request")?; + Ok(token) +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PurgeStats { + pub fragments_rewritten: usize, + pub old_versions: u64, + pub bytes_removed: u64, + /// False while versions from before the rebuild remain. + pub served: bool, +} + +/// Rewrites every fragment that has deleted rows and rebuilds the indices from +/// the rows that remain, then removes the versions from before that through +/// the cleanup job's retention policy with only the latest version kept, and +/// marks `token` served once none of them is left. The rebuild is needed +/// because compaction only remaps an index: an IVF_FLAT index keeps the +/// deleted vector's values in its partitions under a remapped row address. It +/// runs once per request, so a purge that has to wait for its cleanup (a +/// version pinned by a tag, a query lease or one someone named, fails it) +/// retries only the cleanup. Files no version references stay while Lance +/// cannot tell them from a writer's uncommitted ones, which is up to seven +/// days for an abandoned write. +pub async fn purge(dataset: &mut Dataset, token: &str) -> Result { + let fragments_rewritten = rebuild(dataset, token).await?; + let cleanup = match retention_policy(dataset, 0, 1).await? { + Some(policy) => cleanup_old_versions(dataset, policy) + .await + .context("remove versions older than the purge")?, + None => RemovalStats::default(), + }; + let served = finish(dataset, token).await?; + Ok(PurgeStats { + fragments_rewritten, + old_versions: cleanup.old_versions, + bytes_removed: cleanup.bytes_removed, + served, + }) +} + +/// Rewrites the fragments with deletions and rebuilds the indices for +/// `token`, unless that was done already, and returns how many fragments it +/// rewrote. +pub async fn rebuild(dataset: &mut Dataset, token: &str) -> Result { + if dataset.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) == Some(token) { + return Ok(0); + } + let vector_indices = vector_index_names(dataset).await?; + let options = CompactionOptions { + materialize_deletions: true, + // Zero materializes every fragment with any deletion. + materialize_deletions_threshold: 0.0, + ..Default::default() + }; + let compaction = compact_files(dataset, options, None) + .await + .context("rewrite fragments with deletions")?; + dataset.checkout_latest().await?; + if !vector_indices.is_empty() { + let mut options = OptimizeOptions::retrain(); + options.index_names = Some(vector_indices); + dataset + .optimize_indices(&options) + .await + .context("rebuild indices without the deleted rows")?; + } + rebuild_scalar_indices(dataset).await?; + dataset.checkout_latest().await?; + dataset + .update_config([(PURGE_REBUILT_KEY, token)]) + .await + .context("record purge rebuild")?; + Ok(compaction.fragments_removed) +} + +/// Marks `token` served when the oldest version left is one the rebuild for +/// it wrote, so no version from before the rebuild remains. +pub async fn finish(dataset: &mut Dataset, token: &str) -> Result { + let oldest = dataset + .versions() + .await? + .iter() + .map(|v| v.version) + .min() + .context("the table has no versions")?; + let oldest = dataset.checkout_version(oldest).await?; + if oldest.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) != Some(token) { + return Ok(false); + } + dataset + .update_config([(PURGE_COMPLETED_KEY, token)]) + .await + .context("record purge completion")?; + Ok(true) +} + +/// Names of the vector indices. A retrain builds each name over the whole +/// table, so a column whose index is sharded over several names is refused: +/// its shards would overlap. +async fn vector_index_names(dataset: &Dataset) -> Result> { + let mut by_column: BTreeMap, BTreeSet> = BTreeMap::new(); + for index in dataset + .load_indices() + .await? + .iter() + .filter(|i| is_vector(i)) + { + by_column + .entry(index.fields.clone()) + .or_default() + .insert(index.name.clone()); + } + let mut names = Vec::new(); + for shards in by_column.into_values() { + if shards.len() > 1 { + bail!( + "the vector index is sharded over {shards:?}; a purge would rebuild each over the whole table" + ); + } + names.extend(shards); + } + Ok(names) +} + +/// Builds every scalar index again from the rows that remain. Compaction only +/// remaps them: a bitmap index keeps a deleted key, and under stable row ids +/// no scalar index is rewritten at all. +async fn rebuild_scalar_indices(dataset: &mut Dataset) -> Result<()> { + let indices = dataset.load_indices().await?; + let mut rebuilt = BTreeSet::new(); + for index in indices.iter() { + if is_system_index(index) || is_vector(index) || !rebuilt.insert(index.name.clone()) { + continue; + } + let [field] = index.fields[..] else { + bail!("index {} covers {} columns", index.name, index.fields.len()); + }; + let column = dataset.schema().field_path(field)?; + let params = dataset + .open_scalar_index(&column, &index.uuid, &NoOpMetricsCollector) + .await? + .derive_index_params()?; + dataset + .create_index( + &[column.as_str()], + IndexType::Scalar, + Some(index.name.clone()), + ¶ms, + true, + ) + .await + .with_context(|| format!("rebuild index {}", index.name))?; + } + Ok(()) +} + +fn is_vector(index: &IndexMetadata) -> bool { + index + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("VectorIndexDetails")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + const WORKER: Retention = Retention { + retain_hours: 168, + min_versions: 5, + }; + + #[test] + fn table_retention_overrides_within_range() { + assert_eq!( + Retention::for_table(&config(&[]), WORKER, 8760, 1000), + WORKER + ); + assert_eq!( + Retention::for_table( + &config(&[(RETAIN_HOURS_KEY, "0"), (MIN_VERSIONS_KEY, "1")]), + WORKER, + 8760, + 1000 + ), + Retention { + retain_hours: 0, + min_versions: 1 + } + ); + for bad in ["-1", "8761", "x", ""] { + assert_eq!( + Retention::for_table(&config(&[(RETAIN_HOURS_KEY, bad)]), WORKER, 8760, 1000) + .retain_hours, + 168, + "{bad:?}" + ); + } + assert_eq!( + Retention::for_table(&config(&[(MIN_VERSIONS_KEY, "0")]), WORKER, 8760, 1000) + .min_versions, + 5 + ); + } + + #[test] + fn purge_is_pending_until_its_token_is_served() { + assert_eq!(pending(&config(&[])), None); + assert_eq!(pending(&config(&[(PURGE_REQUESTED_KEY, "")])), None); + assert_eq!( + pending(&config(&[(PURGE_REQUESTED_KEY, "t1")])), + Some("t1".to_string()) + ); + assert_eq!( + pending(&config(&[ + (PURGE_REQUESTED_KEY, "t2"), + (PURGE_COMPLETED_KEY, "t1") + ])), + Some("t2".to_string()) + ); + assert_eq!( + pending(&config(&[ + (PURGE_REQUESTED_KEY, "t2"), + (PURGE_COMPLETED_KEY, "t2") + ])), + None + ); + } +} diff --git a/seaweed-worker/crates/lance/tests/purge.rs b/seaweed-worker/crates/lance/tests/purge.rs new file mode 100644 index 0000000000..071f365841 --- /dev/null +++ b/seaweed-worker/crates/lance/tests/purge.rs @@ -0,0 +1,394 @@ +//! A purge removes a deleted vector's bytes from storage: after it, no file +//! under the table holds the vector, its key or its metadata. Runs on a local +//! table, so it needs no cluster. + +use std::path::Path; +use std::sync::Arc; + +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use lance::Dataset; +use lance::dataset::cleanup::{CleanupPolicy, RemovalStats, cleanup_old_versions}; +use lance::index::DatasetIndexExt; +use lance::index::vector::VectorIndexParams; +use lance_index::IndexType; +use lance_linalg::distance::DistanceType; +use weed_lance_worker::jobs::purge; + +const DIM: i32 = 8; +const ROWS: usize = 300; +const MARKER_ROW: usize = 137; +const KEY_MARKER: &str = "purge-key-5f0c2e9a7d41b3c8"; +const METADATA_MARKER: &str = "purge-metadata-a93e1d5b77c2f046"; + +fn marker_vector() -> Vec { + // Values no other row has, so their bytes identify the row. + (0..DIM).map(|i| 31_337.125 + i as f32 * 0.5).collect() +} + +fn batch() -> (Arc, RecordBatch) { + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM), + false, + ), + Field::new("metadata", DataType::Utf8, true), + ])); + let keys: Vec = (0..ROWS) + .map(|i| { + if i == MARKER_ROW { + KEY_MARKER.to_string() + } else { + format!("k{i}") + } + }) + .collect(); + let metadata: Vec = (0..ROWS) + .map(|i| { + if i == MARKER_ROW { + format!("{{\"note\":\"{METADATA_MARKER}\"}}") + } else { + format!("{{\"n\":{i}}}") + } + }) + .collect(); + let vectors = FixedSizeListArray::from_iter_primitive::( + (0..ROWS).map(|i| { + let values: Vec = if i == MARKER_ROW { + marker_vector() + } else { + (0..DIM) + .map(|d| ((i * 7 + d as usize) % 97) as f32) + .collect() + }; + Some(values.into_iter().map(Some).collect::>()) + }), + DIM, + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(keys)), + Arc::new(vectors), + Arc::new(StringArray::from(metadata)), + ], + ) + .unwrap(); + (schema, batch) +} + +fn vector_bytes() -> Vec { + marker_vector() + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect() +} + +/// Each marker found under `dir`, as " in ". +fn markers_in(dir: &Path) -> Vec { + let vector = vector_bytes(); + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in std::fs::read_dir(&path).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + stack.push(entry.path()); + continue; + } + let bytes = std::fs::read(entry.path()).unwrap(); + let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle); + for (name, needle) in [ + ("vector", vector.as_slice()), + ("key", KEY_MARKER.as_bytes()), + ("metadata", METADATA_MARKER.as_bytes()), + ] { + if has(needle) { + let file = entry.path(); + let file = file.strip_prefix(dir).unwrap_or(&file); + found.push(format!("{name} in {}", file.display())); + } + } + } + } + found.sort_unstable(); + found +} + +/// A purge whose cleanup nothing holds back, so only the latest version +/// stays. [`purge::purge`] cleans up through the cleanup job's retention +/// policy, which a deployment may have keep more. +async fn purge_to_latest( + dataset: &mut Dataset, + token: &str, +) -> anyhow::Result<(usize, RemovalStats, bool)> { + let rewritten = purge::rebuild(dataset, token).await?; + let policy = CleanupPolicy { + before_timestamp: Some(chrono::Utc::now()), + before_version: Some(dataset.latest_version_id().await?), + error_if_tagged_old_versions: true, + ..Default::default() + }; + let removed = cleanup_old_versions(dataset, policy).await?; + Ok((rewritten, removed, purge::finish(dataset, token).await?)) +} + +#[tokio::test] +async fn purge_removes_a_deleted_vectors_bytes() { + purge_leaves_no_trace(false).await; +} + +/// Under stable row ids compaction leaves scalar indices as they are, so only +/// a rebuild takes the key out of them. +#[tokio::test] +async fn purge_removes_a_deleted_vectors_bytes_under_stable_row_ids() { + purge_leaves_no_trace(true).await; +} + +async fn purge_leaves_no_trace(stable_row_ids: bool) { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let params = lance::dataset::WriteParams { + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(params), + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(2, DistanceType::L2), + false, + ) + .await + .unwrap(); + dataset + .create_index( + &["key"], + IndexType::BTree, + Some("key_idx".to_string()), + &lance_index::scalar::ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + // A remapped bitmap index still lists the deleted row's value. + dataset + .create_index( + &["metadata"], + IndexType::Bitmap, + Some("metadata_idx".to_string()), + &lance_index::scalar::ScalarIndexParams::for_builtin( + lance_index::scalar::BuiltinIndexType::Bitmap, + ), + false, + ) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + + let before = markers_in(dir.path()); + assert!( + before.iter().any(|m| m.starts_with("vector ")), + "the vector's bytes should be found before deletion, found {before:?}" + ); + + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + assert_eq!(dataset.count_rows(None).await.unwrap(), ROWS - 1); + + // A deletion alone, and the default retention cleanup, leave the bytes: + // the fragment still holds the row and older versions still reference it. + cleanup_old_versions( + &dataset, + CleanupPolicy { + before_timestamp: Some(chrono::Utc::now() - chrono::Duration::hours(168)), + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + markers_in(dir.path()) + .iter() + .any(|m| m.starts_with("vector ")), + "a deleted vector is expected to stay until purged" + ); + + let token = purge::request_purge(&mut dataset).await.unwrap(); + assert_eq!(purge::pending(dataset.config()), Some(token.clone())); + let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap(); + assert!(served); + assert!(rewritten >= 1); + assert!(removed.old_versions >= 1, "{removed:?}"); + + assert_eq!( + markers_in(dir.path()), + Vec::::new(), + "deleted vector's bytes remain in storage after the purge" + ); + assert_eq!(purge::pending(dataset.config()), None); + + // The table is intact: every other row, and the index, still serve. + let reopened = Dataset::open(uri).await.unwrap(); + assert_eq!(reopened.count_rows(None).await.unwrap(), ROWS - 1); + assert_eq!(purge::pending(reopened.config()), None); + assert_eq!(reopened.load_indices().await.unwrap().len(), 3); +} + +/// A version a query lease (a tag) pins still holds the deleted row, so the +/// purge must fail and leave the request pending rather than report success. +/// Once the tag goes, the next purge only cleans up: the rebuild is not done +/// again. +#[tokio::test] +async fn purge_refuses_while_a_tag_pins_old_data() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(2, DistanceType::L2), + false, + ) + .await + .unwrap(); + let pinned = dataset.version().version; + dataset + .tags() + .create( + "weed-vector-query-lease", + lance::dataset::refs::Ref::Version(None, Some(pinned)), + ) + .await + .unwrap(); + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + let token = purge::request_purge(&mut dataset).await.unwrap(); + assert!(purge_to_latest(&mut dataset, &token).await.is_err()); + dataset.checkout_latest().await.unwrap(); + assert_eq!(purge::pending(dataset.config()), Some(token.clone())); + + let index = |dataset: &Dataset| { + let dataset = dataset.clone(); + async move { dataset.load_indices().await.unwrap()[0].uuid } + }; + let rebuilt = index(&dataset).await; + dataset + .tags() + .delete("weed-vector-query-lease") + .await + .unwrap(); + let (rewritten, _, served) = purge_to_latest(&mut dataset, &token).await.unwrap(); + assert!(served); + assert_eq!(rewritten, 0); + assert_eq!( + index(&dataset).await, + rebuilt, + "the index was rebuilt again" + ); + assert_eq!(purge::pending(dataset.config()), None); + assert_eq!(markers_in(dir.path()), Vec::::new()); +} + +/// The request is served only once no version from before the rebuild is +/// left, whatever kept one. +#[tokio::test] +async fn purge_is_served_once_older_versions_are_gone() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap(); + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + let token = purge::request_purge(&mut dataset).await.unwrap(); + assert_eq!(purge::rebuild(&mut dataset, &token).await.unwrap(), 1); + assert!(!purge::finish(&mut dataset, &token).await.unwrap()); + assert_eq!(purge::pending(dataset.config()), Some(token.clone())); + + let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap(); + assert!(served); + assert_eq!(rewritten, 0); + assert!(removed.old_versions >= 1, "{removed:?}"); + assert_eq!(purge::pending(dataset.config()), None); +} + +/// A retrain builds every index name over the whole table, so shards of one +/// vector index would end up overlapping: the purge refuses before touching +/// the table and leaves the request pending. +#[tokio::test] +async fn purge_refuses_a_sharded_vector_index() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let params = lance::dataset::WriteParams { + max_rows_per_file: ROWS / 2, + ..Default::default() + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(params), + ) + .await + .unwrap(); + let fragments: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + assert_eq!(fragments.len(), 2); + for (shard, fragment) in fragments.iter().enumerate() { + dataset + .create_index_builder( + &["vector"], + IndexType::Vector, + &VectorIndexParams::ivf_flat(2, DistanceType::L2), + ) + .name(format!("shard-{shard}")) + .fragments(vec![*fragment]) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + } + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + let token = purge::request_purge(&mut dataset).await.unwrap(); + let version = dataset.version().version; + + let err = purge::purge(&mut dataset, &token).await.unwrap_err(); + assert!(format!("{err:#}").contains("sharded"), "{err:#}"); + dataset.checkout_latest().await.unwrap(); + assert_eq!(dataset.version().version, version, "the table was changed"); + assert_eq!(purge::pending(dataset.config()), Some(token)); +}