diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index e35e830620..c1fc7395ef 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -908,14 +908,9 @@ impl VectorService for VectorServer { 0 => DEFAULT_PAGE, n => n.min(MAX_PAGE), }; + // A purge keeps only the latest version, so lease the one this page reads. + let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?; if segments > 1 { - // Data and metadata come from a second read of this version, so - // lease it as get_vectors does. - let _lease = if columns.len() > 1 { - self.leases.try_pin(&dataset).await.map_err(internal)? - } else { - None - }; let (vectors, next) = list_segment( &dataset, &columns, diff --git a/seaweed-worker/crates/lance/src/jobs/cleanup.rs b/seaweed-worker/crates/lance/src/jobs/cleanup.rs index 83a992736a..77d1666de6 100644 --- a/seaweed-worker/crates/lance/src/jobs/cleanup.rs +++ b/seaweed-worker/crates/lance/src/jobs/cleanup.rs @@ -3,11 +3,12 @@ use std::collections::HashMap; use anyhow::{Context, Result, anyhow}; use async_trait::async_trait; use chrono::{Duration, Utc}; +use lance::Dataset; use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions}; use seaweed_worker_core::config_form::{form, int_or, int_value, number_field}; use seaweed_worker_core::pb::{ - ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted, - JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor, + AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, + JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor, RunDetectionRequest, }; use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; @@ -15,6 +16,7 @@ use tracing::warn; use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; +use crate::jobs::purge::{self, Retention}; use crate::jobs::{clamp, string_list, table_id}; pub const JOB_TYPE: &str = "lance_cleanup_versions"; @@ -28,7 +30,9 @@ const MAX_MIN_VERSIONS: i64 = 1000; /// Lance keeps every version until something removes it. Lance can also do this /// itself through auto-cleanup, so this job is for deployments that would rather -/// the cluster owned the policy than each writer. +/// the cluster owned the policy than each writer. A table may override the +/// retention in its own config, and may ask for a purge (see [`purge`]), which +/// this job carries out ahead of the retention window. pub struct CleanupVersionsHandler { namespace_url: String, storage: dataset::Storage, @@ -68,9 +72,8 @@ impl CleanupVersionsHandler { /// The oldest version that may be removed while still leaving `min_versions` /// behind, or None when the table has no more than the floor. -async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result> { - let mut versions: Vec = table - .dataset +async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result> { + let mut versions: Vec = dataset .versions() .await? .iter() @@ -83,6 +86,33 @@ async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Resul Ok(Some(versions[versions.len() - min_versions])) } +/// The cleanup that keeps `min_versions` and every version younger than +/// `retain_hours`, or None when the floor leaves nothing to remove. A purge +/// cleans up through it too. +pub async fn retention_policy( + dataset: &Dataset, + retain_hours: i64, + min_versions: usize, +) -> Result> { + let Some(floor) = version_floor(dataset, min_versions).await? else { + return Ok(None); + }; + Ok(Some(CleanupPolicy { + before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)), + // should_clean() ANDs its clauses, so a version has to be both older + // than the window and below the floor to go. + before_version: Some(floor), + // Files this dataset cannot account for are left alone: they may + // belong to a writer that has not committed yet, and deleting them + // would corrupt a commit in flight. + delete_unverified: false, + // A tagged version is pinned on purpose, so refuse rather than + // silently dropping what someone named. + error_if_tagged_old_versions: true, + ..Default::default() + })) +} + #[async_trait] impl JobHandler for CleanupVersionsHandler { fn capability(&self) -> JobTypeCapability { @@ -134,6 +164,12 @@ impl JobHandler for CleanupVersionsHandler { defaults.clone(), )), worker_default_values: defaults, + // A purge compacts and retrains, so it gets a build's time. + admin_runtime_defaults: Some(AdminRuntimeDefaults { + job_type_max_runtime_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS, + execution_timeout_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS, + ..Default::default() + }), ..Default::default() } } @@ -181,10 +217,23 @@ impl JobHandler for CleanupVersionsHandler { continue; } }; + let config = table.dataset.config(); + let pending_purge = purge::pending(config); + let floor = Retention::for_table( + config, + Retention { + retain_hours: DEFAULT_RETAIN_HOURS, + min_versions, + }, + MAX_RETAIN_HOURS, + MAX_MIN_VERSIONS, + ) + .min_versions; // Age is decided at execution against the retention window; a table // at or under the floor cannot lose a version whatever its age, so - // proposing one would only produce a job with nothing to do. - if stats.total_versions <= min_versions { + // proposing one would only produce a job with nothing to do. A + // requested purge ignores the floor: it keeps only the latest. + if pending_purge.is_none() && stats.total_versions <= floor { continue; } let mut parameters: HashMap = HashMap::new(); @@ -193,12 +242,21 @@ impl JobHandler for CleanupVersionsHandler { proposal_id: format!("{JOB_TYPE}:{encoded}"), dedupe_key: format!("{JOB_TYPE}:{encoded}"), job_type: JOB_TYPE.to_string(), - summary: format!("Clean up {encoded} ({} versions)", stats.total_versions), - detail: format!( - "{} versions retained, above the {min_versions} floor; \ - those outside the retention window can go", - stats.total_versions - ), + summary: match &pending_purge { + Some(_) => format!("Purge {encoded} ({} versions)", stats.total_versions), + None => format!("Clean up {encoded} ({} versions)", stats.total_versions), + }, + detail: match &pending_purge { + Some(token) => format!( + "purge requested at {token}: rewrite fragments with deletions \ + and keep only the latest version" + ), + None => format!( + "{} versions retained, above the {floor} floor; \ + those outside the retention window can go", + stats.total_versions + ), + }, parameters, ..Default::default() }); @@ -254,14 +312,33 @@ impl JobHandler for CleanupVersionsHandler { ) as usize; let client = self.client(); - let table = dataset::open(&client, &id, &self.storage).await?; + let mut table = dataset::open(&client, &id, &self.storage).await?; let before = table.stats().await?; + if let Some(token) = purge::pending(table.dataset.config()) { + return self + .execute_purge(request, sender, &job.job_id, &mut table, &token) + .await; + } + let Retention { + retain_hours, + min_versions, + } = Retention::for_table( + table.dataset.config(), + Retention { + retain_hours, + min_versions, + }, + MAX_RETAIN_HOURS, + MAX_MIN_VERSIONS, + ); + // The floor is a promise about how much history survives, so it has to // be applied here and not only when the job was proposed: by the time it // runs, versions may have aged past the retention window, and age alone // would take the table below what the operator asked to keep. - let Some(floor) = version_floor(&table, min_versions).await? else { + let Some(policy) = retention_policy(&table.dataset, retain_hours, min_versions).await? + else { sender.send_completed(JobCompleted { request_id: request.request_id.clone(), job_id: job.job_id.clone(), @@ -288,20 +365,6 @@ impl JobHandler for CleanupVersionsHandler { ..Default::default() })?; - let policy = CleanupPolicy { - before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)), - // should_clean() ANDs its clauses, so a version has to be both older - // than the window and below the floor to go. - before_version: Some(floor), - // Files this dataset cannot account for are left alone: they may - // belong to a writer that has not committed yet, and deleting them - // would corrupt a commit in flight. - delete_unverified: false, - // A tagged version is pinned on purpose, so refuse rather than - // silently dropping what someone named. - error_if_tagged_old_versions: true, - ..Default::default() - }; let stats = cleanup_old_versions(&table.dataset, policy) .await .with_context(|| format!("clean up versions of {}", table.location))?; @@ -343,3 +406,73 @@ impl JobHandler for CleanupVersionsHandler { Ok(()) } } + +impl CleanupVersionsHandler { + async fn execute_purge( + &self, + request: &ExecuteJobRequest, + sender: &dyn ExecutionSender, + job_id: &str, + table: &mut dataset::OpenTable, + token: &str, + ) -> Result<()> { + let stats = purge::purge(&mut table.dataset, token) + .await + .with_context(|| format!("purge {}", table.location))?; + if let Some(counters) = &self.metrics { + counters.versions_removed.inc_by(stats.old_versions); + counters.bytes_reclaimed.inc_by(stats.bytes_removed); + } + let mut output: HashMap = HashMap::new(); + output.insert( + "fragments_rewritten".to_string(), + int_value(stats.fragments_rewritten as i64), + ); + output.insert( + "old_versions_removed".to_string(), + int_value(stats.old_versions as i64), + ); + output.insert( + "bytes_removed".to_string(), + int_value(stats.bytes_removed as i64), + ); + sender.send_completed(JobCompleted { + request_id: request.request_id.clone(), + job_id: job_id.to_string(), + job_type: JOB_TYPE.to_string(), + success: true, + result: Some(JobResult { + output_values: output, + summary: if stats.served { + format!( + "purged: rewrote {} fragments, removed {} versions and {} bytes", + stats.fragments_rewritten, stats.old_versions, stats.bytes_removed + ) + } else { + format!( + "purge pending: rewrote {} fragments, removed {} versions and {} bytes; \ + versions from before the rebuild are still retained", + stats.fragments_rewritten, stats.old_versions, stats.bytes_removed + ) + }, + ..Default::default() + }), + ..Default::default() + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_purge_gets_a_builds_execution_timeout() { + let descriptor = CleanupVersionsHandler::new(String::new()).descriptor(); + let defaults = descriptor.admin_runtime_defaults.unwrap(); + assert_eq!(defaults.execution_timeout_seconds, 3600); + assert_eq!(defaults.job_type_max_runtime_seconds, 3600); + assert!(!defaults.enabled); + } +} diff --git a/seaweed-worker/crates/lance/src/jobs/mod.rs b/seaweed-worker/crates/lance/src/jobs/mod.rs index f654a98bab..54bf5b51d5 100644 --- a/seaweed-worker/crates/lance/src/jobs/mod.rs +++ b/seaweed-worker/crates/lance/src/jobs/mod.rs @@ -9,6 +9,7 @@ pub mod cleanup; pub mod compact; pub mod indices; +pub mod purge; pub mod sort; pub mod vector_index; diff --git a/seaweed-worker/crates/lance/src/jobs/purge.rs b/seaweed-worker/crates/lance/src/jobs/purge.rs new file mode 100644 index 0000000000..25b13b047c --- /dev/null +++ b/seaweed-worker/crates/lance/src/jobs/purge.rs @@ -0,0 +1,328 @@ +//! Per-table retention for version cleanup, and purges: compaction that +//! rewrites every fragment with deletions followed by a cleanup that keeps +//! only the latest version, so the bytes of deleted rows leave storage now +//! rather than after the retention window. +//! +//! Both are set in the table's own Lance config, so any client that can write +//! the table can set them and they travel with it: +//! +//! - `seaweedfs.cleanup.retain_hours` and `seaweedfs.cleanup.min_versions` +//! override the worker's retention for this table. +//! - `seaweedfs.purge.requested` holds a request token ([`request_purge`] +//! writes the current time); a purge is pending while it differs from +//! `seaweedfs.purge.completed`, which the purge sets to the token it served. +//! `seaweedfs.purge.rebuilt` records the token whose rebuild is done. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use anyhow::{Context, Result, bail}; +use chrono::Utc; +use lance::Dataset; +use lance::dataset::cleanup::{RemovalStats, cleanup_old_versions}; +use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::optimize::OptimizeOptions; +use lance_index::{IndexType, is_system_index}; +use lance_table::format::IndexMetadata; +use tracing::warn; + +use crate::jobs::cleanup::retention_policy; + +pub const RETAIN_HOURS_KEY: &str = "seaweedfs.cleanup.retain_hours"; +pub const MIN_VERSIONS_KEY: &str = "seaweedfs.cleanup.min_versions"; +pub const PURGE_REQUESTED_KEY: &str = "seaweedfs.purge.requested"; +pub const PURGE_COMPLETED_KEY: &str = "seaweedfs.purge.completed"; +pub const PURGE_REBUILT_KEY: &str = "seaweedfs.purge.rebuilt"; + +/// The retention cleanup applies to one table: the worker's, unless the +/// table's config overrides a value with one inside the worker's range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Retention { + pub retain_hours: i64, + pub min_versions: usize, +} + +impl Retention { + pub fn for_table( + config: &HashMap, + worker: Retention, + max_retain_hours: i64, + max_min_versions: i64, + ) -> Retention { + let read = |key: &str, min: i64, max: i64| -> Option { + let raw = config.get(key)?; + match raw.trim().parse::() { + Ok(value) if (min..=max).contains(&value) => Some(value), + _ => { + warn!("ignoring table config {key}={raw:?}: not an integer in {min}..={max}"); + None + } + } + }; + Retention { + retain_hours: read(RETAIN_HOURS_KEY, 0, max_retain_hours) + .unwrap_or(worker.retain_hours), + min_versions: read(MIN_VERSIONS_KEY, 1, max_min_versions) + .map(|v| v as usize) + .unwrap_or(worker.min_versions), + } + } +} + +/// The request token of a purge the table is waiting for, if any. +pub fn pending(config: &HashMap) -> Option { + let requested = config.get(PURGE_REQUESTED_KEY).map(|s| s.trim())?; + if requested.is_empty() { + return None; + } + match config.get(PURGE_COMPLETED_KEY) { + Some(completed) if completed.trim() == requested => None, + _ => Some(requested.to_string()), + } +} + +/// Records a purge request on the table and returns its token. The next +/// cleanup detection proposes the purge. +pub async fn request_purge(dataset: &mut Dataset) -> Result { + let token = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + dataset + .update_config([(PURGE_REQUESTED_KEY, token.as_str())]) + .await + .context("record purge request")?; + Ok(token) +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PurgeStats { + pub fragments_rewritten: usize, + pub old_versions: u64, + pub bytes_removed: u64, + /// False while versions from before the rebuild remain. + pub served: bool, +} + +/// Rewrites every fragment that has deleted rows and rebuilds the indices from +/// the rows that remain, then removes the versions from before that through +/// the cleanup job's retention policy with only the latest version kept, and +/// marks `token` served once none of them is left. The rebuild is needed +/// because compaction only remaps an index: an IVF_FLAT index keeps the +/// deleted vector's values in its partitions under a remapped row address. It +/// runs once per request, so a purge that has to wait for its cleanup (a +/// version pinned by a tag, a query lease or one someone named, fails it) +/// retries only the cleanup. Files no version references stay while Lance +/// cannot tell them from a writer's uncommitted ones, which is up to seven +/// days for an abandoned write. +pub async fn purge(dataset: &mut Dataset, token: &str) -> Result { + let fragments_rewritten = rebuild(dataset, token).await?; + let cleanup = match retention_policy(dataset, 0, 1).await? { + Some(policy) => cleanup_old_versions(dataset, policy) + .await + .context("remove versions older than the purge")?, + None => RemovalStats::default(), + }; + let served = finish(dataset, token).await?; + Ok(PurgeStats { + fragments_rewritten, + old_versions: cleanup.old_versions, + bytes_removed: cleanup.bytes_removed, + served, + }) +} + +/// Rewrites the fragments with deletions and rebuilds the indices for +/// `token`, unless that was done already, and returns how many fragments it +/// rewrote. +pub async fn rebuild(dataset: &mut Dataset, token: &str) -> Result { + if dataset.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) == Some(token) { + return Ok(0); + } + let vector_indices = vector_index_names(dataset).await?; + let options = CompactionOptions { + materialize_deletions: true, + // Zero materializes every fragment with any deletion. + materialize_deletions_threshold: 0.0, + ..Default::default() + }; + let compaction = compact_files(dataset, options, None) + .await + .context("rewrite fragments with deletions")?; + dataset.checkout_latest().await?; + if !vector_indices.is_empty() { + let mut options = OptimizeOptions::retrain(); + options.index_names = Some(vector_indices); + dataset + .optimize_indices(&options) + .await + .context("rebuild indices without the deleted rows")?; + } + rebuild_scalar_indices(dataset).await?; + dataset.checkout_latest().await?; + dataset + .update_config([(PURGE_REBUILT_KEY, token)]) + .await + .context("record purge rebuild")?; + Ok(compaction.fragments_removed) +} + +/// Marks `token` served when the oldest version left is one the rebuild for +/// it wrote, so no version from before the rebuild remains. +pub async fn finish(dataset: &mut Dataset, token: &str) -> Result { + let oldest = dataset + .versions() + .await? + .iter() + .map(|v| v.version) + .min() + .context("the table has no versions")?; + let oldest = dataset.checkout_version(oldest).await?; + if oldest.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) != Some(token) { + return Ok(false); + } + dataset + .update_config([(PURGE_COMPLETED_KEY, token)]) + .await + .context("record purge completion")?; + Ok(true) +} + +/// Names of the vector indices. A retrain builds each name over the whole +/// table, so a column whose index is sharded over several names is refused: +/// its shards would overlap. +async fn vector_index_names(dataset: &Dataset) -> Result> { + let mut by_column: BTreeMap, BTreeSet> = BTreeMap::new(); + for index in dataset + .load_indices() + .await? + .iter() + .filter(|i| is_vector(i)) + { + by_column + .entry(index.fields.clone()) + .or_default() + .insert(index.name.clone()); + } + let mut names = Vec::new(); + for shards in by_column.into_values() { + if shards.len() > 1 { + bail!( + "the vector index is sharded over {shards:?}; a purge would rebuild each over the whole table" + ); + } + names.extend(shards); + } + Ok(names) +} + +/// Builds every scalar index again from the rows that remain. Compaction only +/// remaps them: a bitmap index keeps a deleted key, and under stable row ids +/// no scalar index is rewritten at all. +async fn rebuild_scalar_indices(dataset: &mut Dataset) -> Result<()> { + let indices = dataset.load_indices().await?; + let mut rebuilt = BTreeSet::new(); + for index in indices.iter() { + if is_system_index(index) || is_vector(index) || !rebuilt.insert(index.name.clone()) { + continue; + } + let [field] = index.fields[..] else { + bail!("index {} covers {} columns", index.name, index.fields.len()); + }; + let column = dataset.schema().field_path(field)?; + let params = dataset + .open_scalar_index(&column, &index.uuid, &NoOpMetricsCollector) + .await? + .derive_index_params()?; + dataset + .create_index( + &[column.as_str()], + IndexType::Scalar, + Some(index.name.clone()), + ¶ms, + true, + ) + .await + .with_context(|| format!("rebuild index {}", index.name))?; + } + Ok(()) +} + +fn is_vector(index: &IndexMetadata) -> bool { + index + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("VectorIndexDetails")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + const WORKER: Retention = Retention { + retain_hours: 168, + min_versions: 5, + }; + + #[test] + fn table_retention_overrides_within_range() { + assert_eq!( + Retention::for_table(&config(&[]), WORKER, 8760, 1000), + WORKER + ); + assert_eq!( + Retention::for_table( + &config(&[(RETAIN_HOURS_KEY, "0"), (MIN_VERSIONS_KEY, "1")]), + WORKER, + 8760, + 1000 + ), + Retention { + retain_hours: 0, + min_versions: 1 + } + ); + for bad in ["-1", "8761", "x", ""] { + assert_eq!( + Retention::for_table(&config(&[(RETAIN_HOURS_KEY, bad)]), WORKER, 8760, 1000) + .retain_hours, + 168, + "{bad:?}" + ); + } + assert_eq!( + Retention::for_table(&config(&[(MIN_VERSIONS_KEY, "0")]), WORKER, 8760, 1000) + .min_versions, + 5 + ); + } + + #[test] + fn purge_is_pending_until_its_token_is_served() { + assert_eq!(pending(&config(&[])), None); + assert_eq!(pending(&config(&[(PURGE_REQUESTED_KEY, "")])), None); + assert_eq!( + pending(&config(&[(PURGE_REQUESTED_KEY, "t1")])), + Some("t1".to_string()) + ); + assert_eq!( + pending(&config(&[ + (PURGE_REQUESTED_KEY, "t2"), + (PURGE_COMPLETED_KEY, "t1") + ])), + Some("t2".to_string()) + ); + assert_eq!( + pending(&config(&[ + (PURGE_REQUESTED_KEY, "t2"), + (PURGE_COMPLETED_KEY, "t2") + ])), + None + ); + } +} diff --git a/seaweed-worker/crates/lance/tests/purge.rs b/seaweed-worker/crates/lance/tests/purge.rs new file mode 100644 index 0000000000..071f365841 --- /dev/null +++ b/seaweed-worker/crates/lance/tests/purge.rs @@ -0,0 +1,394 @@ +//! A purge removes a deleted vector's bytes from storage: after it, no file +//! under the table holds the vector, its key or its metadata. Runs on a local +//! table, so it needs no cluster. + +use std::path::Path; +use std::sync::Arc; + +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use lance::Dataset; +use lance::dataset::cleanup::{CleanupPolicy, RemovalStats, cleanup_old_versions}; +use lance::index::DatasetIndexExt; +use lance::index::vector::VectorIndexParams; +use lance_index::IndexType; +use lance_linalg::distance::DistanceType; +use weed_lance_worker::jobs::purge; + +const DIM: i32 = 8; +const ROWS: usize = 300; +const MARKER_ROW: usize = 137; +const KEY_MARKER: &str = "purge-key-5f0c2e9a7d41b3c8"; +const METADATA_MARKER: &str = "purge-metadata-a93e1d5b77c2f046"; + +fn marker_vector() -> Vec { + // Values no other row has, so their bytes identify the row. + (0..DIM).map(|i| 31_337.125 + i as f32 * 0.5).collect() +} + +fn batch() -> (Arc, RecordBatch) { + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM), + false, + ), + Field::new("metadata", DataType::Utf8, true), + ])); + let keys: Vec = (0..ROWS) + .map(|i| { + if i == MARKER_ROW { + KEY_MARKER.to_string() + } else { + format!("k{i}") + } + }) + .collect(); + let metadata: Vec = (0..ROWS) + .map(|i| { + if i == MARKER_ROW { + format!("{{\"note\":\"{METADATA_MARKER}\"}}") + } else { + format!("{{\"n\":{i}}}") + } + }) + .collect(); + let vectors = FixedSizeListArray::from_iter_primitive::( + (0..ROWS).map(|i| { + let values: Vec = if i == MARKER_ROW { + marker_vector() + } else { + (0..DIM) + .map(|d| ((i * 7 + d as usize) % 97) as f32) + .collect() + }; + Some(values.into_iter().map(Some).collect::>()) + }), + DIM, + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(keys)), + Arc::new(vectors), + Arc::new(StringArray::from(metadata)), + ], + ) + .unwrap(); + (schema, batch) +} + +fn vector_bytes() -> Vec { + marker_vector() + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect() +} + +/// Each marker found under `dir`, as " in ". +fn markers_in(dir: &Path) -> Vec { + let vector = vector_bytes(); + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in std::fs::read_dir(&path).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + stack.push(entry.path()); + continue; + } + let bytes = std::fs::read(entry.path()).unwrap(); + let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle); + for (name, needle) in [ + ("vector", vector.as_slice()), + ("key", KEY_MARKER.as_bytes()), + ("metadata", METADATA_MARKER.as_bytes()), + ] { + if has(needle) { + let file = entry.path(); + let file = file.strip_prefix(dir).unwrap_or(&file); + found.push(format!("{name} in {}", file.display())); + } + } + } + } + found.sort_unstable(); + found +} + +/// A purge whose cleanup nothing holds back, so only the latest version +/// stays. [`purge::purge`] cleans up through the cleanup job's retention +/// policy, which a deployment may have keep more. +async fn purge_to_latest( + dataset: &mut Dataset, + token: &str, +) -> anyhow::Result<(usize, RemovalStats, bool)> { + let rewritten = purge::rebuild(dataset, token).await?; + let policy = CleanupPolicy { + before_timestamp: Some(chrono::Utc::now()), + before_version: Some(dataset.latest_version_id().await?), + error_if_tagged_old_versions: true, + ..Default::default() + }; + let removed = cleanup_old_versions(dataset, policy).await?; + Ok((rewritten, removed, purge::finish(dataset, token).await?)) +} + +#[tokio::test] +async fn purge_removes_a_deleted_vectors_bytes() { + purge_leaves_no_trace(false).await; +} + +/// Under stable row ids compaction leaves scalar indices as they are, so only +/// a rebuild takes the key out of them. +#[tokio::test] +async fn purge_removes_a_deleted_vectors_bytes_under_stable_row_ids() { + purge_leaves_no_trace(true).await; +} + +async fn purge_leaves_no_trace(stable_row_ids: bool) { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let params = lance::dataset::WriteParams { + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(params), + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(2, DistanceType::L2), + false, + ) + .await + .unwrap(); + dataset + .create_index( + &["key"], + IndexType::BTree, + Some("key_idx".to_string()), + &lance_index::scalar::ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + // A remapped bitmap index still lists the deleted row's value. + dataset + .create_index( + &["metadata"], + IndexType::Bitmap, + Some("metadata_idx".to_string()), + &lance_index::scalar::ScalarIndexParams::for_builtin( + lance_index::scalar::BuiltinIndexType::Bitmap, + ), + false, + ) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + + let before = markers_in(dir.path()); + assert!( + before.iter().any(|m| m.starts_with("vector ")), + "the vector's bytes should be found before deletion, found {before:?}" + ); + + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + assert_eq!(dataset.count_rows(None).await.unwrap(), ROWS - 1); + + // A deletion alone, and the default retention cleanup, leave the bytes: + // the fragment still holds the row and older versions still reference it. + cleanup_old_versions( + &dataset, + CleanupPolicy { + before_timestamp: Some(chrono::Utc::now() - chrono::Duration::hours(168)), + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + markers_in(dir.path()) + .iter() + .any(|m| m.starts_with("vector ")), + "a deleted vector is expected to stay until purged" + ); + + let token = purge::request_purge(&mut dataset).await.unwrap(); + assert_eq!(purge::pending(dataset.config()), Some(token.clone())); + let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap(); + assert!(served); + assert!(rewritten >= 1); + assert!(removed.old_versions >= 1, "{removed:?}"); + + assert_eq!( + markers_in(dir.path()), + Vec::::new(), + "deleted vector's bytes remain in storage after the purge" + ); + assert_eq!(purge::pending(dataset.config()), None); + + // The table is intact: every other row, and the index, still serve. + let reopened = Dataset::open(uri).await.unwrap(); + assert_eq!(reopened.count_rows(None).await.unwrap(), ROWS - 1); + assert_eq!(purge::pending(reopened.config()), None); + assert_eq!(reopened.load_indices().await.unwrap().len(), 3); +} + +/// A version a query lease (a tag) pins still holds the deleted row, so the +/// purge must fail and leave the request pending rather than report success. +/// Once the tag goes, the next purge only cleans up: the rebuild is not done +/// again. +#[tokio::test] +async fn purge_refuses_while_a_tag_pins_old_data() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(2, DistanceType::L2), + false, + ) + .await + .unwrap(); + let pinned = dataset.version().version; + dataset + .tags() + .create( + "weed-vector-query-lease", + lance::dataset::refs::Ref::Version(None, Some(pinned)), + ) + .await + .unwrap(); + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + let token = purge::request_purge(&mut dataset).await.unwrap(); + assert!(purge_to_latest(&mut dataset, &token).await.is_err()); + dataset.checkout_latest().await.unwrap(); + assert_eq!(purge::pending(dataset.config()), Some(token.clone())); + + let index = |dataset: &Dataset| { + let dataset = dataset.clone(); + async move { dataset.load_indices().await.unwrap()[0].uuid } + }; + let rebuilt = index(&dataset).await; + dataset + .tags() + .delete("weed-vector-query-lease") + .await + .unwrap(); + let (rewritten, _, served) = purge_to_latest(&mut dataset, &token).await.unwrap(); + assert!(served); + assert_eq!(rewritten, 0); + assert_eq!( + index(&dataset).await, + rebuilt, + "the index was rebuilt again" + ); + assert_eq!(purge::pending(dataset.config()), None); + assert_eq!(markers_in(dir.path()), Vec::::new()); +} + +/// The request is served only once no version from before the rebuild is +/// left, whatever kept one. +#[tokio::test] +async fn purge_is_served_once_older_versions_are_gone() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap(); + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + let token = purge::request_purge(&mut dataset).await.unwrap(); + assert_eq!(purge::rebuild(&mut dataset, &token).await.unwrap(), 1); + assert!(!purge::finish(&mut dataset, &token).await.unwrap()); + assert_eq!(purge::pending(dataset.config()), Some(token.clone())); + + let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap(); + assert!(served); + assert_eq!(rewritten, 0); + assert!(removed.old_versions >= 1, "{removed:?}"); + assert_eq!(purge::pending(dataset.config()), None); +} + +/// A retrain builds every index name over the whole table, so shards of one +/// vector index would end up overlapping: the purge refuses before touching +/// the table and leaves the request pending. +#[tokio::test] +async fn purge_refuses_a_sharded_vector_index() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("index.lance"); + let uri = uri.to_str().unwrap(); + let (schema, batch) = batch(); + let params = lance::dataset::WriteParams { + max_rows_per_file: ROWS / 2, + ..Default::default() + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(params), + ) + .await + .unwrap(); + let fragments: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + assert_eq!(fragments.len(), 2); + for (shard, fragment) in fragments.iter().enumerate() { + dataset + .create_index_builder( + &["vector"], + IndexType::Vector, + &VectorIndexParams::ivf_flat(2, DistanceType::L2), + ) + .name(format!("shard-{shard}")) + .fragments(vec![*fragment]) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + } + dataset + .delete(&format!("key = '{KEY_MARKER}'")) + .await + .unwrap(); + let token = purge::request_purge(&mut dataset).await.unwrap(); + let version = dataset.version().version; + + let err = purge::purge(&mut dataset, &token).await.unwrap_err(); + assert!(format!("{err:#}").contains("sharded"), "{err:#}"); + dataset.checkout_latest().await.unwrap(); + assert_eq!(dataset.version().version, version, "the table was changed"); + assert_eq!(purge::pending(dataset.config()), Some(token)); +}