diff --git a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml index d2aae98dd9..94907fbcfa 100644 --- a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml @@ -272,6 +272,10 @@ spec: --admin={{ template "seaweedfs.fullname" . }}-admin.{{ include "seaweedfs.namespace" . }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \ {{- end }} --namespace="$LANCE_NAMESPACE_URL" \ + --masters={{ include "seaweedfs.masterServerArg" . }} \ + {{- if .Values.filer.filerGroup }} + --filer-group={{ .Values.filer.filerGroup }} \ + {{- end }} {{- if .Values.global.seaweedfs.enableSecurity }} --tls-ca=/usr/local/share/ca-certificates/ca/tls.crt \ --tls-cert=/usr/local/share/ca-certificates/worker/tls.crt \ @@ -294,6 +298,10 @@ spec: - name: ca-cert readOnly: true mountPath: /usr/local/share/ca-certificates/ca/ + {{- /* grpc.client, which the worker dials the masters and filers with */}} + - name: client-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/client/ - name: worker-cert readOnly: true mountPath: /usr/local/share/ca-certificates/worker/ diff --git a/seaweed-vector/Cargo.lock b/seaweed-vector/Cargo.lock index d24b8df03a..a293806dc1 100644 --- a/seaweed-vector/Cargo.lock +++ b/seaweed-vector/Cargo.lock @@ -7143,6 +7143,8 @@ dependencies = [ "hex", "hkdf", "hmac 0.12.1", + "http", + "hyper-util", "jsonwebtoken", "lance", "lance-datafusion", @@ -7153,8 +7155,11 @@ dependencies = [ "moka", "object_store", "prometheus", + "prost", + "protoc-bin-vendored", "reqwest 0.12.28", "roaring", + "rustls", "seaweed-common", "seaweed-worker-core", "seaweed-worker-sort", @@ -7163,8 +7168,16 @@ dependencies = [ "sha2 0.10.9", "sw-client", "tokio", + "tokio-rustls", + "tokio-stream", + "toml", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tower", "tracing", "tracing-subscriber", + "x509-parser", "zstd", ] @@ -7207,8 +7220,6 @@ dependencies = [ "bytes", "clap", "futures", - "http", - "hyper-util", "lance", "lance-arrow", "lance-linalg", @@ -7218,24 +7229,19 @@ dependencies = [ "prost-types", "protoc-bin-vendored", "reqwest 0.12.28", - "rustls", "serde", "serde_json", "sw-client", "tempfile", "tokio", - "tokio-rustls", "tokio-stream", - "toml", "tonic", "tonic-prost", "tonic-prost-build", - "tower", "tracing", "tracing-subscriber", "weed-lance-worker", "weed-vector-core", - "x509-parser", ] [[package]] diff --git a/seaweed-vector/crates/core/src/main.rs b/seaweed-vector/crates/core/src/main.rs index 6299d64740..a8b059e1eb 100644 --- a/seaweed-vector/crates/core/src/main.rs +++ b/seaweed-vector/crates/core/src/main.rs @@ -1,16 +1,16 @@ use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; -use weed_lance_worker::catalog::{NamespaceClient, parse_id}; -use weed_lance_worker::dataset::{self, FallbackOptions}; +use weed_lance_worker::catalog::{NamespaceClient, SigningKey, TableDescription, parse_id}; +use weed_lance_worker::dataset::{self, Storage}; use weed_vector_core::coverage; use weed_vector_core::index; use weed_vector_core::query::{self, Query}; use weed_vector_core::table::VectorTable; /// Mirrors `weed-worker`'s flags where the two touch the same thing: the -/// namespace is WEED_LANCE_NAMESPACE and the fallback storage credentials are -/// the same env vars. +/// namespace that says where a table is, and the masters whose filers hold +/// it. #[derive(Parser, Debug)] #[command(name = "weed-vector", about = "SeaweedFS vector query spike")] struct Args { @@ -22,12 +22,38 @@ struct Args { )] namespace: String, - /// Storage credentials to use where the namespace vends none. - #[arg(long, env = "WEED_S3_ACCESS_KEY")] - access_key: Option, + /// Comma-separated master addresses, whose filers the tables are kept + /// through. A table named by a local URI needs none. + #[arg(long, alias = "master", env = "WEED_MASTERS")] + masters: Option, - #[arg(long, env = "WEED_S3_SECRET_KEY")] - secret_key: Option, + /// The filer group whose filers hold the tables. + #[arg( + long, + alias = "filer.group", + env = "WEED_FILER_GROUP", + default_value = "" + )] + filer_group: String, + + /// Encrypt every chunk written for the volume servers, as `weed s3 + /// -encryptVolumeData` does. + #[arg( + long = "encryptVolumeData", + alias = "encrypt-volume-data", + env = "WEED_ENCRYPT_VOLUME_DATA" + )] + encrypt_volume_data: bool, + + /// The `jwt.filer_signing.key` of the cluster, which the namespace + /// authenticates cluster processes with; security.toml's by default. + #[arg(long, env = "WEED_FILER_SIGNING_KEY")] + filer_signing_key: Option, + + /// The cluster's security.toml; by default the file weed processes find + /// in -config_dir, ., ~/.seaweedfs and /etc/seaweedfs. + #[arg(long, env = "WEED_VECTOR_SECURITY_TOML")] + security_toml: Option, #[command(subcommand)] command: Command, @@ -126,22 +152,33 @@ enum IndexAction { }, } -/// `id` is a delimiter-joined table id; when it looks like a URI (`s3://...`, -/// `/path/...`, `file://...`) the dataset is opened directly, skipping the -/// namespace — useful for a spike run against a local dataset. +/// `id` is a delimiter-joined table id; when it looks like a URI the dataset +/// is opened directly, skipping the namespace: an `s3://` URI on the +/// cluster's storage, a local path (`/path/...`, `file://...`) as it is. async fn open_dataset( client: &NamespaceClient, id: &str, - fallback: &FallbackOptions, + storage: &Storage, ) -> Result { - if id.contains("://") || id.starts_with('/') { - let mut builder = lance::dataset::builder::DatasetBuilder::from_uri(id); - if !fallback.is_empty() { - builder = builder.with_storage_options(fallback.clone()); - } - return builder.load().await.with_context(|| format!("open {id}")); + let on_cluster = id.starts_with("s3://"); + if !on_cluster && (id.contains("://") || id.starts_with('/')) { + return lance::dataset::builder::DatasetBuilder::from_uri(id) + .load() + .await + .with_context(|| format!("open {id}")); } - let table = dataset::open(client, &parse_id(id), fallback).await?; + if storage.0.is_none() { + bail!("{id} is kept on the cluster and needs --masters"); + } + if on_cluster { + let description = TableDescription { + location: id.to_string(), + storage_options: Default::default(), + managed_versioning: false, + }; + return dataset::open_at(&description, storage.wrapper(id).await?).await; + } + let table = dataset::open(client, &parse_id(id), storage).await?; Ok(table.dataset) } @@ -157,12 +194,23 @@ async fn main() -> Result<()> { .init(); let args = Args::parse(); - let mut fallback = FallbackOptions::new(); - if let (Some(access), Some(secret)) = (args.access_key, args.secret_key) { - fallback.insert("aws_access_key_id".to_string(), access); - fallback.insert("aws_secret_access_key".to_string(), secret); - } - let client = NamespaceClient::new(args.namespace).with_fallback(&fallback); + let security = + weed_lance_worker::security::Security::load(args.security_toml.as_deref(), None)?; + let signing_key = SigningKey::new( + args.filer_signing_key + .or_else(|| security.get("jwt.filer_signing.key")), + ); + let client = NamespaceClient::new(args.namespace).with_signing_key(signing_key.as_ref()); + let storage = match &args.masters { + Some(masters) => { + let cluster = + weed_lance_worker::cluster::Cluster::new(masters, &args.filer_group, &security)?; + Storage(Some( + cluster.storage(&security, args.encrypt_volume_data).await?, + )) + } + None => Storage::default(), + }; match args.command { Command::Describe { @@ -170,7 +218,7 @@ async fn main() -> Result<()> { vector_column, key_column, } => { - let dataset = open_dataset(&client, &id, &fallback).await?; + let dataset = open_dataset(&client, &id, &storage).await?; let schema: arrow_schema::Schema = dataset.schema().into(); let resolved = VectorTable::resolve(&dataset, vector_column.as_deref(), key_column.as_deref()); @@ -210,7 +258,7 @@ async fn main() -> Result<()> { vector_column, key_column, } => { - let dataset = open_dataset(&client, &id, &fallback).await?; + let dataset = open_dataset(&client, &id, &storage).await?; let vector = VectorTable::resolve(&dataset, vector_column.as_deref(), key_column.as_deref())?; let coverages = coverage::coverage(&dataset, &vector.vector_column).await?; @@ -222,7 +270,7 @@ async fn main() -> Result<()> { vector_column, key_column, } => { - let dataset = open_dataset(&client, &id, &fallback).await?; + let dataset = open_dataset(&client, &id, &storage).await?; let vector = VectorTable::resolve(&dataset, vector_column.as_deref(), key_column.as_deref())?; for batch in query::get(&dataset, &vector, &keys, None).await? { @@ -242,7 +290,7 @@ async fn main() -> Result<()> { key_column, nprobes, } => { - let opened = open_dataset(&client, &id, &fallback).await?; + let opened = open_dataset(&client, &id, &storage).await?; let dataset = match version { Some(v) => opened.checkout_version(v).await?, None => opened, @@ -286,7 +334,7 @@ async fn main() -> Result<()> { id, vector_column, .. } => (id.clone(), vector_column.clone()), }; - let mut dataset = open_dataset(&client, &id, &fallback).await?; + let mut dataset = open_dataset(&client, &id, &storage).await?; let table = VectorTable::resolve(&dataset, vector_column.as_deref(), None)?; let started = std::time::Instant::now(); match action { @@ -349,6 +397,24 @@ mod tests { } } + #[test] + fn cluster_flags_mirror_the_engine_and_the_worker() { + let args = Args::try_parse_from([ + "weed-vector", + "--masters", + "m:9333", + "--filer-group", + "eu", + "--encryptVolumeData", + "describe", + "t", + ]) + .unwrap(); + assert_eq!(args.masters.as_deref(), Some("m:9333")); + assert_eq!(args.filer_group, "eu"); + assert!(args.encrypt_volume_data); + } + #[test] fn index_build_takes_partitions_and_a_name() { let args = Args::try_parse_from([ diff --git a/seaweed-vector/crates/core/src/maintenance.rs b/seaweed-vector/crates/core/src/maintenance.rs index 4eee54e03e..048015c7f2 100644 --- a/seaweed-vector/crates/core/src/maintenance.rs +++ b/seaweed-vector/crates/core/src/maintenance.rs @@ -178,8 +178,7 @@ async fn reap_expired(dataset: &Dataset, now: u128) { } }) .is_some_and(|(own, stamp)| { - now.saturating_sub(stamp) - > if own { LEASE_TTL } else { FOREIGN_TTL }.as_nanos() + now.saturating_sub(stamp) > if own { LEASE_TTL } else { FOREIGN_TTL }.as_nanos() }); if expired && !held { let _ = dataset.tags().delete(name).await; @@ -379,7 +378,10 @@ mod tests { let foreign = format!("{QUERY_TAG_PREFIX}other-{stale}-foreign"); dataset .tags() - .create(&foreign, Ref::Version(None, Some(dataset.version().version))) + .create( + &foreign, + Ref::Version(None, Some(dataset.version().version)), + ) .await .unwrap(); let lease = pin(&dataset).await.unwrap(); diff --git a/seaweed-vector/crates/server/Cargo.toml b/seaweed-vector/crates/server/Cargo.toml index 1842651822..6fb8743c72 100644 --- a/seaweed-vector/crates/server/Cargo.toml +++ b/seaweed-vector/crates/server/Cargo.toml @@ -36,13 +36,6 @@ moka = { version = "0.12", features = ["sync"] } object_store = { version = "0.13", features = ["aws"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -toml = "0.8" -rustls = "0.23" -tokio-rustls = "0.26" -x509-parser = "0.16" -tower = "0.5" -hyper-util = { version = "0.1", features = ["tokio"] } -http = "1" tokio.workspace = true tokio-stream.workspace = true tracing.workspace = true diff --git a/seaweed-vector/crates/server/src/lib.rs b/seaweed-vector/crates/server/src/lib.rs index 074dcdc697..2215c11b1a 100644 --- a/seaweed-vector/crates/server/src/lib.rs +++ b/seaweed-vector/crates/server/src/lib.rs @@ -13,8 +13,12 @@ pub mod master_pb { tonic::include_proto!("master_pb"); } pub mod master_client; -pub mod security; -pub mod tls; +pub mod security { + pub use weed_lance_worker::security::*; +} +pub mod tls { + pub use weed_lance_worker::grpc_tls::*; +} pub mod pb { pub mod vector { diff --git a/seaweed-vector/crates/server/src/main.rs b/seaweed-vector/crates/server/src/main.rs index 897190c3c4..b41337136e 100644 --- a/seaweed-vector/crates/server/src/main.rs +++ b/seaweed-vector/crates/server/src/main.rs @@ -1,13 +1,10 @@ use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; use anyhow::{Context, Result}; use clap::Parser; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; -use weed_lance_worker::dataset::FallbackOptions; -use weed_lance_worker::direct_read::{Connection, DirectReads, FilerDialer}; +use weed_lance_worker::cluster::Cluster; use weed_vector_server::master_client; use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer; use weed_vector_server::read_cache::ReadCache; @@ -21,15 +18,18 @@ struct Args { /// gRPC listen address. #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")] listen: String, - /// object_store options for deployments whose gateway vends no - /// credentials: key=value, repeatable. Credentials given here are used - /// only with the aws_endpoint given here. - #[arg(long = "storage-option", alias = "storage.option", value_parser = parse_option)] - storage_options: Vec<(String, String)>, /// Comma-separated master addresses. The engine registers with them, so /// the gateway finds it, and asks them for the filers it reads through. #[arg(long, alias = "master", env = "WEED_VECTOR_MASTERS")] masters: String, + /// Encrypt every chunk written for the volume servers, as `weed s3 + /// -encryptVolumeData` does. + #[arg( + long = "encryptVolumeData", + alias = "encrypt-volume-data", + env = "WEED_VECTOR_ENCRYPT_VOLUME_DATA" + )] + encrypt_volume_data: bool, /// The filer group whose filers the engine reads through. #[arg( long, @@ -64,12 +64,6 @@ struct Args { filer_signing_key: Option, } -fn parse_option(s: &str) -> Result<(String, String), String> { - s.split_once('=') - .map(|(k, v)| (k.to_string(), v.to_string())) - .ok_or_else(|| "expected key=value".to_string()) -} - // weed tools take Go-style single-dash long flags; normalize_args converts // any -flag to --flag — the same normalization weed-volume's config.rs does — // so -master and --masters parse the same. The masters field carries the @@ -97,68 +91,6 @@ fn normalize_args(args: Vec) -> Vec { /// A wildcard listen address is not reachable for peers; derive the outbound /// IP toward the first master instead — the address the master sees anyway. -/// How often the masters are asked for the filers. -const FILER_REFRESH: Duration = Duration::from_secs(30); -/// How long startup waits for a filer to be registered. -const FILER_WAIT: Duration = Duration::from_secs(120); - -/// The filers the masters list, waiting for the first to register. -async fn discover_filers( - masters: &[String], - filer_group: &str, - tls: Option<&weed_vector_server::security::GrpcTls>, -) -> Result> { - let deadline = tokio::time::Instant::now() + FILER_WAIT; - loop { - match master_client::list_filers(masters, filer_group, tls).await { - Ok(filers) if !filers.is_empty() => return Ok(filers), - Ok(_) => tracing::warn!("no filer is registered with the masters yet"), - Err(err) => tracing::warn!("listing filers: {err:#}"), - } - if tokio::time::Instant::now() >= deadline { - anyhow::bail!("no filer registered with the masters within {FILER_WAIT:?}"); - } - tokio::time::sleep(Duration::from_secs(3)).await; - } -} - -async fn follow_filers( - reads: DirectReads, - masters: Vec, - filer_group: String, - tls: Option, -) { - loop { - tokio::time::sleep(FILER_REFRESH).await; - match master_client::list_filers(&masters, &filer_group, tls.as_ref()).await { - Ok(filers) if !filers.is_empty() => { - if let Err(err) = reads.set_filers(&filers) { - tracing::warn!("filers {}: {err:#}", filers.join(",")); - } - } - // Keep the last known filers while the masters list none. - Ok(_) => tracing::warn!("the masters list no filer; keeping the known ones"), - Err(err) => tracing::warn!("listing filers: {err:#}"), - } - } -} - -fn master_addresses(masters: &str) -> Result> { - let masters: Vec = masters - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_string) - .collect(); - if masters.is_empty() { - anyhow::bail!("--masters (WEED_VECTOR_MASTERS) lists no master address"); - } - for master in &masters { - master_client::grpc_addr(master)?; - } - Ok(masters) -} - fn advertised_addr(listen: &str, masters: &[String]) -> String { let Some((host, _)) = listen.rsplit_once(':') else { return listen.to_string(); @@ -203,7 +135,6 @@ async fn main() -> Result<()> { if let Some(path) = security.path() { tracing::info!("security settings from {}", path.display()); } - let master_tls = security.grpc_client_tls("grpc.client", "grpc.master")?; let server_tls = security .grpc_server_tls("grpc.vector")? .map(ServerTls::load) @@ -213,56 +144,11 @@ async fn main() -> Result<()> { let listener = tokio::net::TcpListener::bind(&args.listen) .await .with_context(|| format!("listen on {}", args.listen))?; - let masters = master_addresses(&args.masters)?; - // The filers are dialed as the masters are: grpc.client's credentials - // and server allowlist, through the engine's own connector. - let filer_dialer: Option = security - .grpc_client_tls("grpc.client", "grpc.filer")? - .map(|tls| { - Arc::new(move |addr: &str| { - let channel = tonic::transport::Endpoint::new(format!("http://{addr}"))? - .connect_timeout(Duration::from_secs(10)) - .connect_with_connector_lazy(tls::connector(tls.clone())); - Ok(sw_client::FilerClient::from_channel(addr, channel)) - }) as FilerDialer - }); - let conn = Connection { - filers: discover_filers(&masters, &args.filer_group, master_tls.as_ref()).await?, - read_jwt_key: security.jwt_read_key(), - sse: security.sse(), - grpc_tls: None, - https_tls: security.https_client_tls()?, - filer_dialer, - }; - let reads = DirectReads::connect(&conn).await.context("direct reads")?; - tracing::info!( - "reading tables through {reads:?}; filer gRPC {}, volume servers {}; SSE-S3 objects {}", - if conn.filer_dialer.is_some() { - "over TLS" - } else { - "plaintext" - }, - if conn.https_tls.is_some() { - "over HTTPS" - } else { - "over HTTP" - }, - if reads.sse_s3_configured() { - "are readable" - } else { - "need s3.sse.kek or s3.sse.key" - }, - ); - tokio::spawn(follow_filers( - reads.clone(), - masters.clone(), - args.filer_group.clone(), - master_tls.clone(), - )); - let fallback: FallbackOptions = args.storage_options.into_iter().collect(); - let mut service = VectorServer::new(fallback) + let cluster = Cluster::new(&args.masters, &args.filer_group, &security)?; + let reads = cluster.storage(&security, args.encrypt_volume_data).await?; + let mut service = VectorServer::new() .with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get)) - .with_direct_reads(reads); + .with_storage(reads); if args.read_cache_mb > 0 { service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20)); } @@ -276,15 +162,15 @@ async fn main() -> Result<()> { Some(_) => "over TLS with allowed client certificates required", None => "in plaintext", }, - if master_tls.is_some() { "over TLS" } else { "in plaintext" }, + if cluster.master_tls.is_some() { "over TLS" } else { "in plaintext" }, ); // Registered only now, with storage connected and the listener bound, // so the gateway never finds an engine that cannot serve yet. - let advertise = advertised_addr(&args.listen, &masters); + let advertise = advertised_addr(&args.listen, &cluster.masters); tokio::spawn(master_client::keep_connected( - masters.clone(), + cluster.masters.clone(), advertise, - master_tls.clone(), + cluster.master_tls.clone(), )); let router = Server::builder() // Match the gateway's 1 GiB cap — put batches over the tonic 4 MiB @@ -344,11 +230,5 @@ mod tests { #[test] fn the_masters_are_required() { assert!(Args::try_parse_from(["weed-vector"]).is_err()); - assert!(master_addresses("").is_err()); - assert!(master_addresses(" , ").is_err()); - assert_eq!( - master_addresses(" m1:9333,,m2:9333 ").unwrap(), - ["m1:9333", "m2:9333"] - ); } } diff --git a/seaweed-vector/crates/server/src/master_client.rs b/seaweed-vector/crates/server/src/master_client.rs index eede8a9eec..7476e992c7 100644 --- a/seaweed-vector/crates/server/src/master_client.rs +++ b/seaweed-vector/crates/server/src/master_client.rs @@ -7,12 +7,10 @@ use std::time::Duration; use anyhow::{Context, Result}; use tokio::time::sleep; -use tonic::transport::{Channel, Endpoint}; -use crate::master_pb::{ - KeepConnectedRequest, ListClusterNodesRequest, seaweed_client::SeaweedClient, -}; +use crate::master_pb::{KeepConnectedRequest, seaweed_client::SeaweedClient}; use crate::security::GrpcTls; +pub use weed_lance_worker::cluster::{CONNECT_TIMEOUT, connect, grpc_addr}; fn registration(client_addr: &str) -> KeepConnectedRequest { KeepConnectedRequest { @@ -27,36 +25,6 @@ fn registration(client_addr: &str) -> KeepConnectedRequest { } } -/// Masters are configured by their HTTP address; gRPC lives +10000 like the -/// weed conventions (server_address.ToGrpcAddress). -// A master address may carry an explicit gRPC port as `host:http.grpc` — -// use it directly; otherwise the gRPC port is http + 10000. -pub fn grpc_addr(master: &str) -> Result { - let (scheme, rest) = master.split_once("://").unwrap_or(("http", master)); - if let Some((host, port)) = rest.rsplit_once(':') { - if let Some((_, grpc)) = port.split_once('.') { - if let Ok(grpc) = grpc.parse::() { - let grpc = u16::try_from(grpc) - .with_context(|| format!("master {master}: gRPC port {grpc} is past 65535"))?; - return Ok(format!("{scheme}://{host}:{grpc}")); - } - } - if let Ok(port) = port.parse::() { - let grpc = port - .checked_add(10000) - .and_then(|grpc| u16::try_from(grpc).ok()) - .with_context(|| { - format!( - "master {master}: gRPC port {port}+10000 is past 65535; \ - give it explicitly as {host}:{port}." - ) - })?; - return Ok(format!("{scheme}://{host}:{grpc}")); - } - } - Ok(format!("{scheme}://{rest}")) -} - // `host:http.grpc` normalizes to `host:http` so a leader hint naming the // dialed master does not read as a redirect. fn http_addr(master: &str) -> String { @@ -112,31 +80,6 @@ where } } -/// Bounds the credential reads and the connect, TLS handshake included, and -/// then the master's first answer, the 5s wdclient gives a master call; a -/// black-holed or silent master otherwise holds up the other masters. -const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); - -/// A channel to the master's gRPC, over TLS with `grpc.client`'s credentials -/// when the cluster has them. -pub async fn connect(master: &str, tls: Option<&GrpcTls>) -> Result { - let addr = grpc_addr(master)?; - tokio::time::timeout(CONNECT_TIMEOUT, async { - Ok(match tls { - None => Endpoint::new(addr)?.connect().await?, - // The connector does the TLS; tonic adds its own only for an - // https URI, which it refuses without a tls_config. - Some(tls) => { - Endpoint::new(addr.replacen("https://", "http://", 1))? - .connect_with_connector(crate::tls::connector(tls.clone())) - .await? - } - }) - }) - .await - .context("master did not complete the connection")? -} - async fn stream_to( master: &str, client_addr: &str, @@ -261,11 +204,11 @@ mod tests { } } - /// `tests/tls`: a CA and the `engine-a` leaf it signed. + /// The worker crate's `tests/tls`: a CA and the `engine-a` leaf it signed. fn credentials() -> GrpcTls { let fixture = |name: &str| { std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/tls") + .join("../../../seaweed-worker/crates/lance/tests/tls") .join(name) .display() .to_string() @@ -342,74 +285,3 @@ mod tests { assert!(grpc_addr("m1:9333.65536").is_err()); } } - -/// The filers of `filer_group` registered with the masters, as gRPC -/// `host:port`s, in registration order; the masters are all asked, since a -/// follower may know fewer than the leader. An error means no master answered. -pub async fn list_filers( - masters: &[String], - filer_group: &str, - tls: Option<&GrpcTls>, -) -> Result> { - let mut filers = Vec::new(); - let mut answered = false; - let mut last = None; - for master in masters { - match filers_of(master, filer_group, tls).await { - Ok(found) => { - answered = true; - for filer in found { - if !filers.contains(&filer) { - filers.push(filer); - } - } - } - Err(err) => { - tracing::warn!("master {master}: listing filers: {err:#}"); - last = Some(err); - } - } - } - match (answered, last) { - (false, Some(err)) => Err(err), - (false, None) => Err(anyhow::anyhow!("no master given")), - _ => Ok(filers), - } -} - -async fn filers_of(master: &str, filer_group: &str, tls: Option<&GrpcTls>) -> Result> { - let channel = connect(master, tls).await?; - let request = ListClusterNodesRequest { - client_type: "filer".into(), - filer_group: filer_group.into(), - ..Default::default() - }; - let nodes = tokio::time::timeout( - CONNECT_TIMEOUT, - SeaweedClient::new(channel).list_cluster_nodes(request), - ) - .await - .context("master did not answer")?? - .into_inner() - .cluster_nodes; - Ok(nodes - .iter() - .filter_map(|node| match filer_grpc_addr(&node.address) { - Ok(addr) => Some(addr), - Err(err) => { - tracing::warn!("master {master} lists filer {:?}: {err:#}", node.address); - None - } - }) - .collect()) -} - -/// A filer registers as `host:port.grpc`, or `host:port` with its gRPC at -/// port + 10000. -pub fn filer_grpc_addr(address: &str) -> Result { - let addr = grpc_addr(address)?; - Ok(addr - .split_once("://") - .map_or(addr.as_str(), |(_, rest)| rest) - .to_string()) -} diff --git a/seaweed-vector/crates/server/src/read_cache.rs b/seaweed-vector/crates/server/src/read_cache.rs index 8260b1d690..ec6f21bc2b 100644 --- a/seaweed-vector/crates/server/src/read_cache.rs +++ b/seaweed-vector/crates/server/src/read_cache.rs @@ -291,7 +291,6 @@ mod tests { use object_store::ObjectStoreExt; use object_store::memory::InMemory; use tonic::Request; - use weed_lance_worker::dataset::FallbackOptions; fn cached(inner: Arc) -> Arc { ReadCache::new(1 << 20).wrap("memory", inner) @@ -482,7 +481,7 @@ mod tests { async fn cached_reads_follow_writes_and_a_recreated_index() { let bucket = format!("cache-{}", uuid_like()); let location = format!("shared-memory://{bucket}/vector/x"); - let server = VectorServer::new(FallbackOptions::new()) + let server = VectorServer::new() .with_schemes(&["shared-memory"]) .with_read_cache(ReadCache::new(16 << 20)); let table = pb::Table { diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index ed472052b7..68a1dd2038 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -8,19 +8,14 @@ use lance::Dataset; use lance::dataset::ProjectionRequest; use lance::dataset::WriteParams; use lance::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched}; -use lance::io::ObjectStoreParams; -use lance::io::StorageOptionsAccessor; use lance::io::WrappingObjectStore; use lance::table::format::Fragment; -use object_store::aws::AmazonS3ConfigKey; use prost_types::{ListValue, Struct, Value, value::Kind}; use std::collections::{BTreeMap, HashMap}; -use std::ffi::OsString; -use std::str::FromStr; use std::sync::Arc; use tonic::{Request, Response, Status}; use weed_lance_worker::catalog::TableDescription; -use weed_lance_worker::dataset::{self, FallbackOptions}; +use weed_lance_worker::dataset; use weed_lance_worker::direct_read::{DirectReads, Wrappers}; use weed_lance_worker::jobs::vector_index::is_vector_table; use weed_vector_core::index; @@ -60,70 +55,6 @@ const KEY_COLUMN: &str = "key"; const METADATA_COLUMN: &str = "metadata"; const VECTOR_COLUMN: &str = "vector"; const ROW_ID_COLUMN: &str = "_rowid"; -const ENDPOINT_OPTION: &str = "aws_endpoint"; -/// The options a gateway's describe produces. Any other key a request sends, -/// such as a proxy or an endpoint alias, could route the engine's credentials -/// elsewhere, so it is not taken from the request. -const SENT_OPTIONS: [&str; 8] = [ - ENDPOINT_OPTION, - "allow_http", - "aws_region", - "aws_skip_signature", - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - "expires_at_millis", -]; -const CREDENTIAL_OPTIONS: [&str; 3] = [ - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", -]; - -/// Whether a request's options may be used with the engine's fallback: known -/// keys only, and an endpoint that is the configured one, or any when no -/// credential of the engine's can reach it: none configured, and the request -/// either skips signing or signs with its own credentials alone. `env` names -/// the engine's environment variables. -fn options_trusted( - sent: &HashMap, - fallback: &FallbackOptions, - env: impl IntoIterator, -) -> bool { - if sent.keys().any(|key| !SENT_OPTIONS.contains(&key.as_str())) { - return false; - } - match (sent.get(ENDPOINT_OPTION), fallback.get(ENDPOINT_OPTION)) { - (None, _) => true, - (Some(sent), Some(pinned)) => sent.trim_end_matches('/') == pinned.trim_end_matches('/'), - (Some(_), None) => { - let configured = CREDENTIAL_OPTIONS - .iter() - .any(|key| fallback.contains_key(*key)); - let unsigned = sent.get("aws_skip_signature").is_some_and(|v| v == "true"); - !configured && (unsigned || signs_alone(sent, env)) - } - } -} - -/// Whether lance signs with the request's credentials alone. Lance fills an -/// option the request leaves out from an environment variable under any of -/// object_store's names for it, and asks the AWS default chain when the key -/// pair is incomplete. -fn signs_alone(sent: &HashMap, env: impl IntoIterator) -> bool { - let has = |key: &str| sent.contains_key(key); - let env_token = || { - env.into_iter().any(|name| { - name.to_str() - .and_then(|name| AmazonS3ConfigKey::from_str(&name.to_ascii_lowercase()).ok()) - == Some(AmazonS3ConfigKey::Token) - }) - }; - has("aws_access_key_id") - && has("aws_secret_access_key") - && (has("aws_session_token") || !env_token()) -} - fn schema_for( dimension: usize, metric: pb::DistanceMetric, @@ -440,6 +371,15 @@ fn rows_as_vectors( /// any other location would point the engine at storage the catalog does not /// know. fn resolved(table: &pb::Table, schemes: &[&str]) -> Option { + // Each id part is one path component; `..` would resolve the colocated + // path elsewhere on the filer. + if !table + .id + .iter() + .all(|part| !part.is_empty() && part != "." && part != ".." && !part.contains('/')) + { + return None; + } let colocated = dataset::table_location(&table.id)?; let path = colocated.strip_prefix("s3://")?; let location = table.location.trim_end_matches('/'); @@ -449,25 +389,29 @@ fn resolved(table: &pb::Table, schemes: &[&str]) -> Option { .any(|scheme| location == format!("{scheme}://{path}")); trusted.then(|| TableDescription { location: location.to_string(), - storage_options: table.storage_options.clone(), + storage_options: dataset::unsigned_options(), managed_versioning: false, }) } pub struct VectorServer { - fallback: FallbackOptions, - direct_reads: Option, + storage: Option, read_cache: Option, nprobes: Option, /// URI schemes a table's location may have. schemes: &'static [&'static str], } +impl Default for VectorServer { + fn default() -> Self { + Self::new() + } +} + impl VectorServer { - pub fn new(fallback: FallbackOptions) -> Self { + pub fn new() -> Self { Self { - fallback, - direct_reads: None, + storage: None, read_cache: None, nprobes: None, schemes: &["s3"], @@ -487,9 +431,10 @@ impl VectorServer { self } - /// Read tables from the filers and volume servers instead of the gateway. - pub fn with_direct_reads(mut self, reads: DirectReads) -> Self { - self.direct_reads = Some(reads); + /// Keep tables on the filers and volume servers, not through the gateway. + /// Unset only for tests on an in-process store. + pub fn with_storage(mut self, storage: DirectReads) -> Self { + self.storage = Some(storage); self } @@ -513,47 +458,26 @@ impl VectorServer { table.id.join("/") )) })?; - let env = std::env::vars_os().map(|(name, _)| name); - if !options_trusted(&description.storage_options, &self.fallback, env) { - // The gateway answers AccessDenied without logging, so this is - // where an operator sees a pinned endpoint that does not match. - let endpoint = |options: &HashMap| { - options.get(ENDPOINT_OPTION).cloned().unwrap_or_default() - }; - tracing::warn!( - index = %table.id.join("/"), - sent = %endpoint(&description.storage_options), - pinned = %endpoint(&self.fallback), - "refused the request's storage options: an aws_endpoint other than the pinned one, a key other than the gateway's, or credentials the engine would have to complete" - ); - return Err(Status::permission_denied( - "the storage options would send the engine's credentials elsewhere", - )); - } Ok(description) } + /// The store Lance opens a table with: direct storage innermost, the read + /// cache outside it so a hit never reaches storage. + fn wrapper(&self, table: &pb::Table) -> Option> { + let mut wrappers: Vec> = Vec::new(); + if let Some(storage) = &self.storage { + wrappers.push(Arc::new(storage.encrypting(table.sse_s3))); + } + if let Some(cache) = &self.read_cache { + wrappers.push(Arc::new(cache.clone())); + } + (!wrappers.is_empty()).then(|| Arc::new(Wrappers(wrappers)) as Arc) + } + async fn open(&self, table: &pb::Table) -> Result { let description = self.describe(table)?; - // Direct reads innermost; the cache sits outside them, so a hit never - // reaches storage. The options name the store the bytes come from - // when reads go through the gateway, so the cache is scoped by it and - // the same bucket and path on another endpoint never share. - let mut wrappers: Vec> = Vec::new(); - if let Some(reads) = &self.direct_reads { - wrappers.push(Arc::new(reads.clone())); - } - if let Some(cache) = &self.read_cache { - let options = dataset::table_options(&description, &self.fallback); - let endpoint = options - .get(ENDPOINT_OPTION) - .map(String::as_str) - .unwrap_or(""); - wrappers.push(Arc::new(cache.scoped(endpoint))); - } - let wrapper = (!wrappers.is_empty()) - .then(|| Arc::new(Wrappers(wrappers)) as Arc); - let mut dataset = dataset::open_at(&description, &self.fallback, wrapper) + let wrapper = self.wrapper(table); + let mut dataset = dataset::open_at(&description, wrapper) .await .map_err(open_status)?; if table.version != 0 { @@ -585,7 +509,6 @@ impl VectorService for VectorServer { } // The gateway has already created the table through the catalog. let description = self.describe(&table)?; - let options = dataset::table_options(&description, &self.fallback); let schema = schema_for(dimension, metric, &req.non_filterable_metadata_keys); let empty = RecordBatch::new_empty(schema.clone()); let reader = arrow_array::RecordBatchIterator::new(vec![Ok(empty)].into_iter(), schema); @@ -593,12 +516,7 @@ impl VectorService for VectorServer { reader, &description.location, Some(WriteParams { - store_params: Some(ObjectStoreParams { - storage_options_accessor: Some(Arc::new( - StorageOptionsAccessor::with_static_options(options), - )), - ..Default::default() - }), + store_params: Some(dataset::store_params(self.wrapper(&table))), ..Default::default() }), ) @@ -617,16 +535,24 @@ impl VectorService for VectorServer { let table = req .table .ok_or_else(|| Status::invalid_argument("missing table"))?; + // describe() already refused any location but the index's own. let description = self.describe(&table)?; - dataset::drop_at(&table.id, &description, &self.fallback) - .await - .map_err(|err| { - if err.is::() { - Status::failed_precondition(format!("{err:#}")) - } else { - internal(err) - } - })?; + match &self.storage { + Some(storage) => storage + .remove_table(&description.location) + .await + .map_err(internal)?, + None => { + let (store, path) = lance::io::ObjectStore::from_uri_and_params( + Arc::new(lance::io::ObjectStoreRegistry::default()), + &description.location, + &dataset::store_params(self.wrapper(&table)), + ) + .await + .map_err(internal)?; + store.remove_dir_all(path).await.map_err(internal)?; + } + } Ok(Response::new(pb::DropIndexResponse {})) } @@ -1093,7 +1019,7 @@ mod tests { /// opened or removed. #[tokio::test] async fn drop_index_refuses_a_location_outside_the_table() { - let server = VectorServer::new(FallbackOptions::new()); + let server = VectorServer::new(); for location in [ "s3://bkt/vector", "s3://bkt/vector/prod", @@ -1120,7 +1046,6 @@ mod tests { let table = |id: &[&str], location: &str| pb::Table { id: id.iter().map(|s| s.to_string()).collect(), location: location.into(), - storage_options: HashMap::from([("aws_endpoint".to_string(), "http://gw".to_string())]), ..Default::default() }; let s3 = &["s3"]; @@ -1131,9 +1056,9 @@ mod tests { assert_eq!( description .storage_options - .get("aws_endpoint") + .get("aws_skip_signature") .map(String::as_str), - Some("http://gw") + Some("true") ); } assert!(resolved(&table(&index, ""), s3).is_none()); @@ -1142,13 +1067,15 @@ mod tests { assert!(resolved(&table(&index, "file://bkt/vector/x"), s3).is_none()); assert!(resolved(&table(&index, "memory://bkt/vector/x"), &["s3", "memory"]).is_some()); assert!(resolved(&table(&["bkt", "ns", "x"], "s3://bkt/ns/x"), s3).is_none()); + assert!(resolved(&table(&["bkt", "vector", ".."], "s3://bkt/vector/.."), s3).is_none()); + assert!(resolved(&table(&["bkt", "vector", "a/b"], "s3://bkt/vector/a/b"), s3).is_none()); assert!(resolved(&table(&["bkt", "x"], "s3://bkt/x"), s3).is_none()); } /// A request names the table's location or is refused. #[tokio::test] async fn a_table_without_its_location_is_refused() { - let server = VectorServer::new(FallbackOptions::new()); + let server = VectorServer::new(); for (location, code) in [ ("", tonic::Code::InvalidArgument), ("s3://other/vector/x", tonic::Code::PermissionDenied), @@ -1168,131 +1095,4 @@ mod tests { assert_eq!(err.code(), code, "{location}"); } } - - /// A request's options are used with the engine's credentials only when - /// they cannot send those credentials elsewhere. - #[test] - fn options_are_trusted_only_where_credentials_cannot_leave() { - let options = |pairs: &[(&str, &str)]| -> HashMap { - pairs - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() - }; - let creds = options(&[("aws_access_key_id", "k"), ("aws_secret_access_key", "s")]); - let pinned = options(&[ - ("aws_access_key_id", "k"), - ("aws_secret_access_key", "s"), - (ENDPOINT_OPTION, "http://gw"), - ]); - let gateway = options(&[(ENDPOINT_OPTION, "http://gw"), ("allow_http", "true")]); - let elsewhere = options(&[(ENDPOINT_OPTION, "http://elsewhere")]); - let unsigned = options(&[ - (ENDPOINT_OPTION, "http://gw"), - ("aws_skip_signature", "true"), - ]); - let vended = options(&[ - (ENDPOINT_OPTION, "http://gw"), - ("aws_access_key_id", "v"), - ("aws_secret_access_key", "v"), - ]); - let none = FallbackOptions::new(); - let trusted = |sent: &HashMap, fallback: &FallbackOptions| { - options_trusted(sent, fallback, Vec::::new()) - }; - - assert!(trusted(&gateway, &pinned)); - assert!(trusted( - &options(&[(ENDPOINT_OPTION, "http://gw/")]), - &pinned - )); - assert!(!trusted(&elsewhere, &pinned)); - assert!(!trusted(&gateway, &creds), "no configured endpoint"); - assert!(!trusted(&gateway, &none), "the default chain could sign"); - assert!(trusted(&unsigned, &none)); - assert!(trusted(&vended, &none)); - assert!( - !options_trusted(&vended, &none, [OsString::from("AWS_SESSION_TOKEN")]), - "the environment would add its session token" - ); - assert!( - !trusted( - &options(&[(ENDPOINT_OPTION, "http://gw"), ("aws_access_key_id", "v")]), - &none - ), - "a key without its secret is signed by the default chain" - ); - assert!( - !trusted(&vended, &creds), - "configured credentials stay home" - ); - assert!( - trusted(&options(&[("aws_region", "r")]), &creds), - "no endpoint sent" - ); - for key in ["proxy_url", "aws_endpoint_url_s3", "aws_endpoint_url"] { - let routed = options(&[(key, "http://elsewhere"), ("aws_skip_signature", "true")]); - assert!(!trusted(&routed, &none), "{key}"); - } - } - - /// A request signs alone only with a full key pair, and with its own - /// session token whenever the environment would supply one. - #[test] - fn a_request_signs_alone_without_the_environment() { - let sent = |pairs: &[(&str, &str)]| -> HashMap { - pairs - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() - }; - let env = |names: &[&str]| -> Vec { names.iter().map(OsString::from).collect() }; - let pair = sent(&[("aws_access_key_id", "v"), ("aws_secret_access_key", "v")]); - let with_token = sent(&[ - ("aws_access_key_id", "v"), - ("aws_secret_access_key", "v"), - ("aws_session_token", "t"), - ]); - - assert!(signs_alone(&pair, env(&["AWS_REGION", "HOME"]))); - assert!(!signs_alone(&sent(&[("aws_access_key_id", "v")]), env(&[]))); - assert!(!signs_alone( - &sent(&[("aws_secret_access_key", "v")]), - env(&[]) - )); - for name in [ - "AWS_SESSION_TOKEN", - "aws_session_token", - "TOKEN", - "SESSION_TOKEN", - ] { - assert!(!signs_alone(&pair, env(&[name])), "{name}"); - assert!(signs_alone(&with_token, env(&[name])), "{name}"); - } - } - - /// Options the engine must not use are refused, for a drop as for an open. - #[tokio::test] - async fn untrusted_options_are_refused() { - let sent = |endpoint: &str| pb::Table { - id: vec!["bkt".into(), "vector".into(), "x".into()], - location: "s3://bkt/vector/x".into(), - storage_options: HashMap::from([(ENDPOINT_OPTION.to_string(), endpoint.to_string())]), - ..Default::default() - }; - let pinned = - FallbackOptions::from([(ENDPOINT_OPTION.to_string(), "http://gw".to_string())]); - let server = VectorServer::new(pinned); - let description = server.describe(&sent("http://gw")).unwrap(); - assert_eq!(description.location, "s3://bkt/vector/x"); - let err = server.describe(&sent("http://elsewhere")).unwrap_err(); - assert_eq!(err.code(), tonic::Code::PermissionDenied); - let err = server - .drop_index(Request::new(pb::DropIndexRequest { - table: Some(sent("http://elsewhere")), - })) - .await - .unwrap_err(); - assert_eq!(err.code(), tonic::Code::PermissionDenied); - } } diff --git a/seaweed-worker/Cargo.lock b/seaweed-worker/Cargo.lock index f87272e123..979a25696b 100644 --- a/seaweed-worker/Cargo.lock +++ b/seaweed-worker/Cargo.lock @@ -383,6 +383,45 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -1581,6 +1620,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "datafusion" version = "54.1.0" @@ -2230,6 +2275,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -3408,7 +3467,7 @@ dependencies = [ "fast-float2", "itoa", "jiff", - "nom", + "nom 8.0.0", "num-traits", "ordered-float", "rand 0.9.5", @@ -4258,6 +4317,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -4320,6 +4385,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -4473,6 +4548,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -5626,6 +5710,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustix" version = "1.1.4" @@ -7075,6 +7168,8 @@ dependencies = [ "hex", "hkdf", "hmac 0.12.1", + "http 1.5.0", + "hyper-util", "jsonwebtoken", "lance", "lance-datafusion", @@ -7085,8 +7180,11 @@ dependencies = [ "moka", "object_store", "prometheus", + "prost", + "protoc-bin-vendored", "reqwest 0.12.28", "roaring", + "rustls", "seaweed-common", "seaweed-worker-core", "seaweed-worker-sort", @@ -7094,10 +7192,18 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sw-client", + "tempfile", "tokio", + "tokio-rustls", + "tokio-stream", + "toml", "tonic", + "tonic-prost", + "tonic-prost-build", + "tower", "tracing", "tracing-subscriber", + "x509-parser", "zstd", ] @@ -7281,6 +7387,23 @@ dependencies = [ "tap", ] +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index d9a40ea6af..c92c89b3ba 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -35,7 +35,7 @@ binary for the runner's platform, resolved from the version in `Cargo.lock`. ## Running - cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 + cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 --masters 127.0.0.1:9333 The admin's *HTTP* address is what an operator has; the gRPC port is derived from it the way the Go side does. Dialling the HTTP port fails as "frame with @@ -49,7 +49,7 @@ Released builds do not need a toolchain. The worker ships inside the SeaweedFS image, beside the Rust volume server, under the verb that mirrors `volume-rust`: - docker run chrislusf/seaweedfs worker-rust --admin admin:12535 + docker run chrislusf/seaweedfs worker-rust --admin admin:12535 --masters master:9333 and as `weed-worker_linux_{amd64,arm64}.tar.gz` on each GitHub release. Both are linux amd64/arm64 only — lance, arrow and datafusion make every extra target an @@ -59,7 +59,7 @@ entrypoint says so rather than failing as "not found". ## Metrics - cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 --metrics-port 9328 + cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 --masters 127.0.0.1:9333 --metrics-port 9328 Serves `/health`, `/ready` and `/metrics` on that port, the same three the Go worker serves under `weed worker -metricsPort`, so one scrape config covers @@ -89,19 +89,21 @@ running but is not going to do anything. ## Credentials -The worker holds none. It asks the namespace to describe a table with -`vend_credentials` and hands the `storage_options` that come back to lance. A -gateway without STS configured vends no credentials at all, so `--access-key` -and `--secret-key` supply a fallback; anything the namespace does vend wins over -them. +The worker holds no S3 credentials. The namespace tells it where a table is; +its files are read and written on the cluster's filers and volume servers, +which `--masters` names the masters of (`--filer-group` picks the filers). What +that takes comes from the same security.toml the weed processes use: +`grpc.client` for the filers, `https.client` for volume servers under HTTPS, +`jwt.signing.read.key` where volume reads are signed, and `s3.sse.kek` or +`s3.sse.key` where a bucket encrypts with SSE-S3. `--encryptVolumeData` encrypts +every chunk the worker writes for the volume servers, as the gateway's flag does. -In a vector bucket's `vector` namespace the gateway vends write credentials only -to a caller that proves it holds `jwt.filer_signing.key`. The same key signs -the admin stream when admin requires it. The worker reads it from where -`weed worker` does: `WEED_JWT_FILER_SIGNING_KEY`, or `[jwt.filer_signing]` in -the security.toml found in the working directory, `~/.seaweedfs`, -`/usr/local/etc/seaweedfs` or `/etc/seaweedfs`. `--filer-signing-key` overrides -both. +The namespace authenticates the worker by the cluster token it derives from +`jwt.filer_signing.key`; the same key signs the admin stream when admin requires +it. The worker reads it from where `weed worker` does: `WEED_JWT_FILER_SIGNING_KEY`, +or `[jwt.filer_signing]` in the security.toml found in the working directory, +`~/.seaweedfs`, `/usr/local/etc/seaweedfs` or `/etc/seaweedfs`. +`--filer-signing-key` overrides both. ## State @@ -113,7 +115,10 @@ All three jobs are implemented and tested end to end against a live gateway: `cargo test -p weed-lance-worker` runs them when `WEED_LANCE_NAMESPACE` names a live namespace and skips otherwise, the way the Go integration tests skip -without Docker. Each test seeds the table it needs, including building a vector +without Docker. A live namespace needs `WEED_MASTERS` naming its cluster's +masters, and `WEED_JWT_FILER_SIGNING_KEY` (or security.toml) where the +namespace authenticates cluster processes; the tests fail without the first +and are refused without the second. Each test seeds the table it needs, including building a vector index and then appending rows outside it, so a run does not depend on what the previous one left behind — the first version of these did, and quietly stopped testing anything once it had done its job. diff --git a/seaweed-worker/crates/lance/Cargo.toml b/seaweed-worker/crates/lance/Cargo.toml index edc660f2e2..ccc224ad94 100644 --- a/seaweed-worker/crates/lance/Cargo.toml +++ b/seaweed-worker/crates/lance/Cargo.toml @@ -67,14 +67,32 @@ sha2 = "0.10" hkdf = "0.12" flate2 = "1" zstd = "0.13" +# security.toml and the gRPC TLS weed processes use with it: the client +# connector with the server allowlist, and the acceptor the engine serves with. +toml = "0.8" +rustls = "0.23" +tokio-rustls = "0.26" +x509-parser = "0.16" +tower = "0.5" +hyper-util = { version = "0.1", features = ["tokio"] } +http = "1" +tonic.workspace = true +tonic-prost.workspace = true +prost.workspace = true +tokio-stream.workspace = true + +[build-dependencies] +tonic-prost-build.workspace = true +protoc-bin-vendored = "3" [dev-dependencies] -tokio = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } tonic.workspace = true arrow-array = "58" arrow-schema = "58" arrow-cast = "58" lance-linalg = "10" +tempfile = "3" [lints] workspace = true diff --git a/seaweed-worker/crates/lance/build.rs b/seaweed-worker/crates/lance/build.rs new file mode 100644 index 0000000000..950cb67886 --- /dev/null +++ b/seaweed-worker/crates/lance/build.rs @@ -0,0 +1,29 @@ +fn main() -> Result<(), Box> { + // Use the protoc that ships with protoc-bin-vendored rather than a system + // one, so the build needs no package manager and always sees the same + // version. An explicit PROTOC still wins, for packagers supplying their own + // and for the lance crates, whose own build scripts read the same variable. + if std::env::var_os("PROTOC").is_none() { + // SAFETY: a build script's main runs single-threaded before anything + // else in this process, so no other thread can be reading the + // environment concurrently. + unsafe { + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?); + } + } + // The master's ListClusterNodes, for the filers; master.proto brings the + // two it imports. + tonic_prost_build::configure() + .build_server(false) + .build_client(true) + .compile_protos( + &[ + "../../../weed/pb/master.proto", + "../../../weed/pb/volume_server.proto", + "../../../weed/pb/remote.proto", + ], + &["../../../weed/pb/"], + )?; + println!("cargo:rerun-if-changed=../../../weed/pb/master.proto"); + Ok(()) +} diff --git a/seaweed-worker/crates/lance/src/catalog/namespace.rs b/seaweed-worker/crates/lance/src/catalog/namespace.rs index bbbfda6f0f..9f141885b5 100644 --- a/seaweed-worker/crates/lance/src/catalog/namespace.rs +++ b/seaweed-worker/crates/lance/src/catalog/namespace.rs @@ -67,7 +67,6 @@ struct ListTablesResponse { pub struct NamespaceClient { base_url: String, http: reqwest::Client, - api_key: Option, signing_key: Option, } @@ -104,29 +103,10 @@ impl NamespaceClient { Self { base_url: base_url.into().trim_end_matches('/').to_string(), http, - api_key: None, signing_key: None, } } - /// The catalog accepts an S3 credential as `x-api-key`; where the worker - /// already holds one for storage it authenticates the namespace calls too. - pub fn with_fallback(mut self, fallback: &crate::dataset::FallbackOptions) -> Self { - if let (Some(access), Some(secret)) = ( - fallback.get("aws_access_key_id"), - fallback.get("aws_secret_access_key"), - ) { - if !protected_transport(&self.base_url) { - tracing::warn!( - "catalog url {} exposes the x-api-key credential in cleartext", - self.base_url - ); - } - self.api_key = Some(format!("{access}:{secret}")); - } - self - } - pub fn with_signing_key(mut self, signing_key: Option<&SigningKey>) -> Self { if signing_key.is_some() && !protected_transport(&self.base_url) { tracing::warn!( @@ -139,10 +119,6 @@ impl NamespaceClient { } fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - let request = match &self.api_key { - Some(key) => request.header("x-api-key", key), - None => request, - }; match self.signing_key.as_ref().and_then(SigningKey::token) { Some(token) => request.header(CLUSTER_TOKEN_HEADER, token), None => request, diff --git a/seaweed-worker/crates/lance/src/cluster.rs b/seaweed-worker/crates/lance/src/cluster.rs new file mode 100644 index 0000000000..aa8f79ad99 --- /dev/null +++ b/seaweed-worker/crates/lance/src/cluster.rs @@ -0,0 +1,296 @@ +//! The cluster as a Lance process joins it: the masters dialed as weed +//! clients dial them, the filers they list, and the storage connection built +//! from them and security.toml that the engine, the worker and the CLI share. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use tonic::transport::{Channel, Endpoint}; + +use crate::direct_read::{Connection, DirectReads, FilerDialer}; +use crate::grpc_tls; +use crate::master_pb::{ListClusterNodesRequest, seaweed_client::SeaweedClient}; +use crate::security::{GrpcTls, Security}; + +/// Masters are configured by their HTTP address; gRPC lives +10000 like the +/// weed conventions (server_address.ToGrpcAddress). +// A master address may carry an explicit gRPC port as `host:http.grpc` — +// use it directly; otherwise the gRPC port is http + 10000. +pub fn grpc_addr(master: &str) -> Result { + let (scheme, rest) = master.split_once("://").unwrap_or(("http", master)); + if let Some((host, port)) = rest.rsplit_once(':') { + if let Some((_, grpc)) = port.split_once('.') { + if let Ok(grpc) = grpc.parse::() { + let grpc = u16::try_from(grpc) + .with_context(|| format!("master {master}: gRPC port {grpc} is past 65535"))?; + return Ok(format!("{scheme}://{host}:{grpc}")); + } + } + if let Ok(port) = port.parse::() { + let grpc = port + .checked_add(10000) + .and_then(|grpc| u16::try_from(grpc).ok()) + .with_context(|| { + format!( + "master {master}: gRPC port {port}+10000 is past 65535; \ + give it explicitly as {host}:{port}." + ) + })?; + return Ok(format!("{scheme}://{host}:{grpc}")); + } + } + Ok(format!("{scheme}://{rest}")) +} + +/// Bounds the credential reads and the connect, TLS handshake included, and +/// then the master's first answer, the 5s wdclient gives a master call; a +/// black-holed or silent master otherwise holds up the other masters. +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// A channel to the master's gRPC, over TLS with `grpc.client`'s credentials +/// when the cluster has them. +pub async fn connect(master: &str, tls: Option<&GrpcTls>) -> Result { + let addr = grpc_addr(master)?; + tokio::time::timeout(CONNECT_TIMEOUT, async { + Ok(match tls { + None => Endpoint::new(addr)?.connect().await?, + // The connector does the TLS; tonic adds its own only for an + // https URI, which it refuses without a tls_config. + Some(tls) => { + Endpoint::new(addr.replacen("https://", "http://", 1))? + .connect_with_connector(grpc_tls::connector(tls.clone())) + .await? + } + }) + }) + .await + .context("master did not complete the connection")? +} + +/// The filers of `filer_group` registered with the masters, as gRPC +/// `host:port`s, in registration order; the masters are all asked, since a +/// follower may know fewer than the leader. An error means no master answered. +pub async fn list_filers( + masters: &[String], + filer_group: &str, + tls: Option<&GrpcTls>, +) -> Result> { + let mut filers = Vec::new(); + let mut answered = false; + let mut last = None; + for master in masters { + match filers_of(master, filer_group, tls).await { + Ok(found) => { + answered = true; + for filer in found { + if !filers.contains(&filer) { + filers.push(filer); + } + } + } + Err(err) => { + tracing::warn!("master {master}: listing filers: {err:#}"); + last = Some(err); + } + } + } + match (answered, last) { + (false, Some(err)) => Err(err), + (false, None) => Err(anyhow::anyhow!("no master given")), + _ => Ok(filers), + } +} + +async fn filers_of(master: &str, filer_group: &str, tls: Option<&GrpcTls>) -> Result> { + let channel = connect(master, tls).await?; + let request = ListClusterNodesRequest { + client_type: "filer".into(), + filer_group: filer_group.into(), + ..Default::default() + }; + let nodes = tokio::time::timeout( + CONNECT_TIMEOUT, + SeaweedClient::new(channel).list_cluster_nodes(request), + ) + .await + .context("master did not answer")?? + .into_inner() + .cluster_nodes; + Ok(nodes + .iter() + .filter_map(|node| match filer_grpc_addr(&node.address) { + Ok(addr) => Some(addr), + Err(err) => { + tracing::warn!("master {master} lists filer {:?}: {err:#}", node.address); + None + } + }) + .collect()) +} + +/// A filer registers as `host:port.grpc`, or `host:port` with its gRPC at +/// port + 10000. +pub fn filer_grpc_addr(address: &str) -> Result { + let addr = grpc_addr(address)?; + Ok(addr + .split_once("://") + .map_or(addr.as_str(), |(_, rest)| rest) + .to_string()) +} + +/// How often the masters are asked for the filers. +const FILER_REFRESH: Duration = Duration::from_secs(30); +/// How long startup waits for a filer to be registered. +const FILER_WAIT: Duration = Duration::from_secs(120); + +/// The filers the masters list, waiting for the first to register. +async fn discover_filers( + masters: &[String], + filer_group: &str, + tls: Option<&GrpcTls>, +) -> Result> { + let deadline = tokio::time::Instant::now() + FILER_WAIT; + loop { + match list_filers(masters, filer_group, tls).await { + Ok(filers) if !filers.is_empty() => return Ok(filers), + Ok(_) => tracing::warn!("no filer is registered with the masters yet"), + Err(err) => tracing::warn!("listing filers: {err:#}"), + } + if tokio::time::Instant::now() >= deadline { + anyhow::bail!("no filer registered with the masters within {FILER_WAIT:?}"); + } + tokio::time::sleep(Duration::from_secs(3)).await; + } +} + +async fn follow_filers( + reads: DirectReads, + masters: Vec, + filer_group: String, + tls: Option, +) { + loop { + tokio::time::sleep(FILER_REFRESH).await; + match list_filers(&masters, &filer_group, tls.as_ref()).await { + Ok(filers) if !filers.is_empty() => { + if let Err(err) = reads.set_filers(&filers) { + tracing::warn!("filers {}: {err:#}", filers.join(",")); + } + } + // Keep the last known filers while the masters list none. + Ok(_) => tracing::warn!("the masters list no filer; keeping the known ones"), + Err(err) => tracing::warn!("listing filers: {err:#}"), + } + } +} + +/// The comma-separated masters, each with a valid gRPC address. +pub fn master_addresses(masters: &str) -> Result> { + let masters: Vec = masters + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + if masters.is_empty() { + anyhow::bail!("the master list names no master address"); + } + for master in &masters { + grpc_addr(master)?; + } + Ok(masters) +} + +/// What the cluster's storage is reached with: the filers the masters list, +/// dialed as the masters are, and the keys security.toml holds. +pub struct Cluster { + pub masters: Vec, + pub filer_group: String, + /// `grpc.client`'s credentials for the masters, when the cluster has them. + pub master_tls: Option, +} + +impl Cluster { + pub fn new(masters: &str, filer_group: &str, security: &Security) -> Result { + Ok(Self { + masters: master_addresses(masters)?, + filer_group: filer_group.to_string(), + master_tls: security.grpc_client_tls("grpc.client", "grpc.master")?, + }) + } + + /// Connects to the cluster's storage and follows the masters' filer list. + pub async fn storage( + &self, + security: &Security, + encrypt_volume_data: bool, + ) -> Result { + // The filers are dialed as the masters are: grpc.client's credentials + // and server allowlist, through this crate's own connector. + let filer_dialer: Option = security + .grpc_client_tls("grpc.client", "grpc.filer")? + .map(|tls| { + Arc::new(move |addr: &str| { + let channel = Endpoint::new(format!("http://{addr}"))? + .connect_timeout(Duration::from_secs(10)) + .connect_with_connector_lazy(grpc_tls::connector(tls.clone())); + Ok(sw_client::FilerClient::from_channel(addr, channel)) + }) as FilerDialer + }); + let conn = Connection { + filers: discover_filers(&self.masters, &self.filer_group, self.master_tls.as_ref()) + .await?, + read_jwt_key: security.jwt_read_key(), + sse: security.sse(), + grpc_tls: None, + https_tls: security.https_client_tls()?, + filer_dialer, + encrypt_volume_data, + filer_group: self.filer_group.clone(), + }; + let storage = DirectReads::connect(&conn) + .await + .context("direct storage")?; + tracing::info!( + "tables kept through {storage:?}; filer gRPC {}, volume servers {}; SSE-S3 objects {}", + if conn.filer_dialer.is_some() { + "over TLS" + } else { + "plaintext" + }, + if conn.https_tls.is_some() { + "over HTTPS" + } else { + "over HTTP" + }, + if storage.sse_s3_configured() { + "are readable" + } else { + "need s3.sse.kek or s3.sse.key" + }, + ); + tokio::spawn(follow_filers( + storage.clone(), + self.masters.clone(), + self.filer_group.clone(), + self.master_tls.clone(), + )); + Ok(storage) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_empty_master_list_is_refused() { + assert!(master_addresses("").is_err()); + assert!(master_addresses(" , ").is_err()); + assert_eq!( + master_addresses(" m1:9333,,m2:9333 ").unwrap(), + ["m1:9333", "m2:9333"] + ); + } +} diff --git a/seaweed-worker/crates/lance/src/dataset.rs b/seaweed-worker/crates/lance/src/dataset.rs index e3cd6d0f0b..93056056c1 100644 --- a/seaweed-worker/crates/lance/src/dataset.rs +++ b/seaweed-worker/crates/lance/src/dataset.rs @@ -1,9 +1,9 @@ -//! Opening a Lance dataset with credentials the namespace vended. +//! Opening a Lance dataset on the cluster's storage. //! -//! The worker holds no storage credentials of its own: it asks the namespace to -//! describe a table with `vend_credentials`, and the `storage_options` that come -//! back are handed to lance as-is. They expire, so a job that runs longer than -//! their lifetime re-describes rather than carrying one set throughout. +//! The namespace says where a table is; its files are read and written on the +//! filers and volume servers through [`DirectReads`]. The S3 store lance builds +//! for the location is unsigned and never dialed, so no process here holds S3 +//! credentials. Tests on an in-process store open without storage. use std::collections::HashMap; use std::sync::Arc; @@ -17,6 +17,7 @@ use lance::io::{ }; use crate::catalog::{NamespaceClient, TableDescription}; +use crate::direct_read::DirectReads; /// A table the worker is about to work on. pub struct OpenTable { @@ -37,27 +38,43 @@ pub struct TableStats { pub schema: Option, } -/// Storage options an operator supplies for deployments that vend none. -pub type FallbackOptions = HashMap; +/// Where tables are kept: the cluster's filers and volume servers, or, for +/// tests on an in-process store, nowhere in particular. +#[derive(Clone, Default)] +pub struct Storage(pub Option); -/// Resolve a table through the namespace and open it. -pub async fn open( - client: &NamespaceClient, - id: &[String], - fallback: &FallbackOptions, -) -> Result { - open_with(client, id, fallback, None).await +impl Storage { + /// The store wrapper for a table at `location`, writing as the table's + /// bucket declares. + pub async fn wrapper(&self, location: &str) -> Result>> { + let Some(direct) = &self.0 else { + return Ok(None); + }; + let bucket = location + .strip_prefix("s3://") + .and_then(|rest| rest.split('/').next()) + .with_context(|| format!("{location} names no bucket"))?; + let sse_s3 = direct.bucket_encrypts(bucket).await?; + Ok(Some(Arc::new(direct.encrypting(sse_s3)))) + } } -/// [`open`] with the dataset's object store wrapped, e.g. by a read cache. -pub async fn open_with( - client: &NamespaceClient, - id: &[String], - fallback: &FallbackOptions, - wrapper: Option>, -) -> Result { +/// What lance builds its own S3 store with: unsigned, so it looks for no +/// credentials, and never dialed. +pub fn unsigned_options() -> HashMap { + HashMap::from([ + ("aws_skip_signature".to_string(), "true".to_string()), + ("aws_region".to_string(), "us-east-1".to_string()), + ("aws_access_key_id".to_string(), String::new()), + ("aws_secret_access_key".to_string(), String::new()), + ]) +} + +/// Resolve a table through the namespace and open it. +pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) -> Result { let description = client.describe_table(id).await?; - let dataset = open_at(&description, fallback, wrapper).await?; + let wrapper = storage.wrapper(&description.location).await?; + let dataset = open_at(&description, wrapper).await?; Ok(OpenTable { id: id.to_vec(), location: description.location, @@ -65,83 +82,51 @@ pub async fn open_with( }) } -/// The object_store options for a table. The namespace vends object_store's own -/// option names, so they pass straight through. What it vends always wins: the -/// fallback exists because a deployment without STS vends no credentials at -/// all, and then the worker has no other way to reach the data. -pub fn table_options( - description: &TableDescription, - fallback: &FallbackOptions, -) -> HashMap { - let mut options: HashMap = fallback.clone(); - options.extend(description.storage_options.clone()); - // A store that takes unsigned requests needs no credentials, but without - // any lance builds the AWS default chain, and that probes instance - // metadata on every open. - if options - .get("aws_skip_signature") - .is_some_and(|v| v == "true") - { - options.entry("aws_access_key_id".into()).or_default(); - options.entry("aws_secret_access_key".into()).or_default(); - } - options -} - -/// Open a table the caller has already resolved. +/// Open a table the caller has already resolved, with the store wrapped. pub async fn open_at( description: &TableDescription, - fallback: &FallbackOptions, wrapper: Option>, ) -> Result { DatasetBuilder::from_uri(&description.location) - .with_store_params(ObjectStoreParams { - storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - table_options(description, fallback), - ))), - object_store_wrapper: wrapper, - ..Default::default() - }) + .with_store_params(store_params(wrapper)) .load() .await .with_context(|| format!("open lance dataset at {}", description.location)) } +/// The store parameters every open and write uses. +pub fn store_params(wrapper: Option>) -> ObjectStoreParams { + ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + unsigned_options(), + ))), + object_store_wrapper: wrapper, + ..Default::default() + } +} + /// Remove every object under a table's location. A table the catalog can no /// longer describe is already dropped, so the call stays idempotent. -pub async fn drop( - client: &NamespaceClient, - id: &[String], - fallback: &FallbackOptions, -) -> Result<()> { +pub async fn drop(client: &NamespaceClient, id: &[String], storage: &Storage) -> Result<()> { let description = match client.describe_table(id).await { Ok(description) => description, Err(err) if describe_not_found(&err) => return Ok(()), Err(err) => return Err(err), }; - drop_at(id, &description, fallback).await -} - -/// [`drop`] for a table the caller has already resolved. -pub async fn drop_at( - id: &[String], - description: &TableDescription, - fallback: &FallbackOptions, -) -> Result<()> { check_drop_location(id, &description.location)?; - let (store, path) = ObjectStore::from_uri_and_params( - Arc::new(ObjectStoreRegistry::default()), - &description.location, - &ObjectStoreParams { - storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - table_options(description, fallback), - ))), - ..Default::default() - }, - ) - .await?; - store.remove_dir_all(path).await?; - Ok(()) + match &storage.0 { + Some(direct) => direct.remove_table(&description.location).await, + None => { + let (store, path) = ObjectStore::from_uri_and_params( + Arc::new(ObjectStoreRegistry::default()), + &description.location, + &store_params(None), + ) + .await?; + store.remove_dir_all(path).await?; + Ok(()) + } + } } /// Where the catalog puts a table it was given no location for, as the @@ -228,45 +213,12 @@ fn schema_json(dataset: &Dataset) -> Option { #[cfg(test)] mod tests { - use super::{FallbackOptions, check_drop_location, table_location, table_options}; - use crate::catalog::TableDescription; + use super::{check_drop_location, table_location}; fn id(parts: &[&str]) -> Vec { parts.iter().map(|part| part.to_string()).collect() } - fn options(pairs: &[(&str, &str)]) -> FallbackOptions { - pairs - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() - } - - #[test] - fn table_options_keep_an_unsigned_store_off_the_aws_default_chain() { - let described = |vended: &[(&str, &str)]| TableDescription { - location: "s3://bkt/vector/x".into(), - storage_options: options(vended), - managed_versioning: false, - }; - let unsigned = described(&[("aws_skip_signature", "true")]); - - let got = table_options(&unsigned, &FallbackOptions::new()); - assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("")); - assert_eq!( - got.get("aws_secret_access_key").map(String::as_str), - Some("") - ); - - let fallback = options(&[("aws_access_key_id", "k"), ("aws_secret_access_key", "s")]); - let got = table_options(&unsigned, &fallback); - assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("k")); - - let signed = described(&[("aws_endpoint", "http://s3:8333")]); - let got = table_options(&signed, &FallbackOptions::new()); - assert!(!got.contains_key("aws_access_key_id"), "{got:?}"); - } - #[test] fn table_location_matches_the_gateway_layout() { assert_eq!( diff --git a/seaweed-worker/crates/lance/src/direct_read.rs b/seaweed-worker/crates/lance/src/direct_read.rs index eee786869b..b1f09ea341 100644 --- a/seaweed-worker/crates/lance/src/direct_read.rs +++ b/seaweed-worker/crates/lance/src/direct_read.rs @@ -1,8 +1,11 @@ -//! Reads of a table straight from the filer and the volume servers, skipping -//! the S3 gateway: object bytes, metadata and listings alike, whatever the -//! entry looks like (inline content, chunk manifests, sparse or overlapping -//! chunks, encryption by the filer or by the gateway, objects tiered to -//! remote storage). Writes, copies and deletes still go to the gateway. +//! A table's storage straight on the filer and the volume servers, skipping +//! the S3 gateway. Reads serve object bytes, metadata and listings alike, +//! whatever the entry looks like (inline content, chunk manifests, sparse or +//! overlapping chunks, encryption by the filer or by the gateway, objects +//! tiered to remote storage). Writes upload chunks to the volume servers the +//! filer assigns and create the entry as the gateway would, encrypting as the +//! bucket and the cluster ask; a conditional create is the filer's exclusive +//! create. Copies and renames are not needed by Lance and not offered. use std::fmt; use std::ops::Range; @@ -20,18 +23,23 @@ use futures::stream::BoxStream; use lance::io::WrappingObjectStore; use md5::{Digest, Md5}; use object_store::path::Path; +use object_store::{Attribute, Attributes, PutMode, UploadPart}; use object_store::{ CopyOptions, Error, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result, }; -use sw_client::filer_pb::Entry; +use sw_client::filer_pb::{Entry, FileChunk, FuseAttributes}; use sw_client::{ - ChunkDecoder, FilerClient, GrpcTls, HttpTransport, HttpsTls, ReadOptions, ResolvedChunks, - VolumeReader, VolumeTransport, + AssignVolumeOptions, Assignment, ChunkDecoder, FilerClient, GrpcTls, HttpTransport, HttpsTls, + ReadOptions, ResolvedChunks, VolumeReader, VolumeTransport, VolumeWriteReceipt, Writer, }; -use crate::sse::{EntryDecoder, KEK_PATH, SseConfig, SseS3Keys}; +use crate::sse::{EntryDecoder, EntryEncoder, KEK_PATH, SseConfig, SseS3Keys}; + +/// `filer_pb.SSEType.SSE_S3`, as the chunks of an SSE-S3 object record it. +#[cfg(test)] +const SSE_S3_TYPE: i32 = 3; const OBJECT_CACHE_ENTRIES: u64 = 1 << 16; const READ_JWT_TTL_SECS: u64 = 60; @@ -41,12 +49,27 @@ const FILER_TIMEOUT: Duration = Duration::from_secs(30); /// The gateway's object-level ETag, set when it differs from the entry's md5. const ETAG_ATTRIBUTE: &str = "Seaweed-X-Amz-ETag"; const PREFIX_OBJECT_ATTRIBUTE: &str = "x-seaweedfs-prefix-object"; +/// What the gateway records for an object it wrote. +const OCTET_STREAM: &str = "application/octet-stream"; +const FILE_MODE: u32 = 0o660; +/// Chunk size when the filer names none. +const DEFAULT_CHUNK_SIZE: usize = 4 << 20; +/// The S3 Tables record of a table bucket, on the bucket's entry. +const TABLE_BUCKET_METADATA: &str = "s3tables.metadata"; #[derive(Clone)] pub struct DirectReads { shared: Arc, /// Set when the source is the cluster's filers, for [`Self::set_filers`]. filers: Option>, + /// Objects written are encrypted with SSE-S3, as the bucket asks. + sse_s3: bool, +} + +/// A chunk uploaded to the volume server the filer assigned. +struct Uploaded { + chunk: FileChunk, + assignment: Assignment, } /// An object as the filer holds it, ready to read. @@ -73,6 +96,26 @@ trait Source: Send + Sync { decoder: Arc, ) -> anyhow::Result; async fn read(&self, object: &Object, offset: u64, len: u64) -> anyhow::Result>; + /// Asks the filer for a volume server to hold a needle of `size` bytes + /// for the file at `path`, in `collection`. + async fn assign(&self, path: &str, collection: &str, size: u64) -> anyhow::Result; + /// Writes `data` to its assigned volume server as the chunk of `path` at + /// `offset`. + async fn write( + &self, + assignment: &Assignment, + path: &str, + offset: u64, + data: Bytes, + ) -> anyhow::Result; + /// Creates `entry` in `dir`; `exclusive` refuses an entry already there + /// with [`sw_client::EntryExists`]. + async fn create(&self, dir: &str, entry: Entry, exclusive: bool) -> anyhow::Result<()>; + /// Removes `name` in `dir` and its data, a directory's content too when + /// `recursive`; a missing entry is no error. + async fn delete(&self, dir: &str, name: &str, recursive: bool) -> anyhow::Result<()>; + /// Removes a needle whose entry was never created. + async fn discard(&self, assignment: &Assignment) -> anyhow::Result<()>; fn name(&self) -> String; } @@ -83,6 +126,32 @@ struct Filers { dial: FilerDialer, current: AtomicUsize, volumes: VolumeReader, + transport: Arc, + writer: Writer, +} + +/// Marks a filer call whose outcome is unknown: the filer took it and never +/// answered. Detect it with [`outcome_uncertain`]. +#[derive(Debug)] +struct NoAnswer; + +impl fmt::Display for NoAnswer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("the filer did not answer") + } +} + +impl std::error::Error for NoAnswer {} + +/// Whether a failed filer write may have been applied all the same. +fn outcome_uncertain(err: &anyhow::Error) -> bool { + err.downcast_ref::().is_some() || sw_client::is_unavailable(err) +} + +/// Whether `name` is one path component: no separator, not empty, not `.` +/// or `..`, which a filer path would resolve elsewhere. +fn plain_name(name: &str) -> bool { + !name.is_empty() && name != "." && name != ".." && !name.contains('/') } /// Connects to a filer by its gRPC `host:port`. @@ -118,6 +187,7 @@ impl Filers { } let start = self.current.load(Ordering::Relaxed); let mut last = None; + let mut unanswered = false; for attempt in 0..clients.len() { let i = (start + attempt) % clients.len(); match tokio::time::timeout(FILER_TIMEOUT, op(clients[i].clone())).await { @@ -125,23 +195,31 @@ impl Filers { self.current.store(i, Ordering::Relaxed); return Ok(value); } + // A filer that fell away may have taken the request first. Ok(Err(err)) if sw_client::is_unavailable(&err) => { tracing::warn!( "filer {} unavailable for {what}: {err:#}", clients[i].addr() ); + unanswered = true; last = Some(err); } + // A filer passed over without an answer may still apply it. + Ok(Err(err)) if unanswered => return Err(err.context(NoAnswer)), Ok(Err(err)) => return Err(err), Err(_) => { + unanswered = true; tracing::warn!( "filer {} did not answer {what} within {FILER_TIMEOUT:?}", clients[i].addr() ); - last = Some(anyhow::anyhow!( - "filer {} did not answer {what} within {FILER_TIMEOUT:?}", - clients[i].addr() - )); + last = Some( + anyhow::anyhow!( + "filer {} did not answer {what} within {FILER_TIMEOUT:?}", + clients[i].addr() + ) + .context(NoAnswer), + ); } } } @@ -249,6 +327,68 @@ impl Source for Filers { } } + async fn assign(&self, path: &str, collection: &str, size: u64) -> anyhow::Result { + // The filer picks replication from the path's storage rules, and the + // collection too when none is named, as it does for the gateway. + let options = AssignVolumeOptions { + collection: collection.to_string(), + ..Default::default() + }; + self.with("assign", |mut c| { + let (path, options) = (path.to_string(), options.clone()); + Box::pin(async move { c.assign_volume_with_options(&path, size, &options).await }) + }) + .await + } + + async fn write( + &self, + assignment: &Assignment, + path: &str, + offset: u64, + data: Bytes, + ) -> anyhow::Result { + let name = path.rsplit('/').next().unwrap_or(path).to_string(); + let digest = BASE64_STANDARD.encode(Md5::digest(&data)); + let mut chunk = self + .writer + .upload_assigned_chunk_bytes(assignment, &name, offset as i64, data) + .await?; + chunk.e_tag = digest; + Ok(chunk) + } + + async fn create(&self, dir: &str, entry: Entry, exclusive: bool) -> anyhow::Result<()> { + self.with("create", |mut c| { + let (dir, entry) = (dir.to_string(), entry.clone()); + Box::pin(async move { + c.create_entry(&dir, entry, 0, exclusive, false) + .await + .map(|_| ()) + }) + }) + .await + } + + async fn delete(&self, dir: &str, name: &str, recursive: bool) -> anyhow::Result<()> { + self.with("delete", |mut c| { + let (dir, name) = (dir.to_string(), name.to_string()); + Box::pin(async move { + match c.delete_entry(&dir, &name, recursive, true, 0).await { + Err(err) if is_not_found(&err) => Ok(()), + other => other, + } + }) + }) + .await + } + + async fn discard(&self, assignment: &Assignment) -> anyhow::Result<()> { + self.transport + .delete(&assignment.url, &assignment.file_id, &assignment.auth) + .await + } + fn name(&self) -> String { self.clients .read() @@ -261,7 +401,9 @@ impl Source for Filers { } /// The filer answers a listing of a missing directory with its missing-entry -/// error; the filer client's tonic is not this crate's, so it is read as text. +/// error, `filer_pb.ErrNotFound`; the filer client's tonic is not this +/// crate's, so it is read as text. Other errors saying "not found", such as +/// an unresolved remote storage, are failures. fn is_not_found(err: &anyhow::Error) -> bool { err.chain() .any(|cause| cause.to_string().contains("no entry is found")) @@ -281,12 +423,22 @@ struct Stats { lookups: AtomicU64, listings: AtomicU64, failures: AtomicU64, + writes: AtomicU64, + written: AtomicU64, + deletes: AtomicU64, } struct Shared { source: Arc, buckets_dir: String, keys: Arc, + /// Bytes per uploaded chunk, as the filer configures writers. + chunk_size: usize, + /// Chunks are encrypted for the volume servers, as the cluster asks. + cipher: bool, + /// The filer group, which prefixes a bucket's collection as the gateway + /// names it. + filer_group: String, /// Write-once files, resolved once; everything else is looked up per /// request so reads follow writes. objects: moka::sync::Cache>, @@ -320,6 +472,11 @@ pub struct Connection { /// Dials the filers in place of `grpc_tls`, for a TLS setup sw-client /// cannot express, such as a server allowlist. pub filer_dialer: Option, + /// Encrypt every chunk written for the volume servers, as `weed s3 + /// -encryptVolumeData` does; the filer's own setting turns it on too. + pub encrypt_volume_data: bool, + /// The filer group, which the gateway prefixes a bucket's collection with. + pub filer_group: String, } impl fmt::Debug for Connection { @@ -331,6 +488,8 @@ impl fmt::Debug for Connection { .field("grpc_tls", &self.grpc_tls) .field("https_tls", &self.https_tls) .field("filer_dialer", &self.filer_dialer.is_some()) + .field("encrypt_volume_data", &self.encrypt_volume_data) + .field("filer_group", &self.filer_group) .finish() } } @@ -364,7 +523,9 @@ impl DirectReads { dial, current: AtomicUsize::new(0), // Lance caches above this store, so needles are not kept here. - volumes: VolumeReader::with_transport_capped(transport, 0), + volumes: VolumeReader::with_transport_capped(transport.clone(), 0), + writer: Writer::with_transport(transport.clone()), + transport, }); filers.set_addresses(&conn.filers)?; let config = filers @@ -378,8 +539,18 @@ impl DirectReads { .context("looking up the filer's SSE-S3 KEK")? .map(|entry| entry.content); let keys = SseS3Keys::load(&conn.sse, kek_file.as_deref())?; + let chunk_size = match config.max_mb { + 0 => DEFAULT_CHUNK_SIZE, + mb => (mb as usize) << 20, + }; let mut reads = Self::new(filers.clone(), &config.dir_buckets, Arc::new(keys)); reads.filers = Some(filers); + { + let shared = Arc::get_mut(&mut reads.shared).expect("just created"); + shared.chunk_size = chunk_size; + shared.cipher = config.cipher || conn.encrypt_volume_data; + shared.filer_group = conn.filer_group.clone(); + } reads.report_every(Duration::from_secs(30)); Ok(reads) } @@ -394,13 +565,88 @@ impl DirectReads { source, buckets_dir, keys, - objects: moka::sync::Cache::new(OBJECT_CACHE_ENTRIES), + chunk_size: DEFAULT_CHUNK_SIZE, + cipher: false, + filer_group: String::new(), + objects: moka::sync::Cache::builder() + .max_capacity(OBJECT_CACHE_ENTRIES) + .support_invalidation_closures() + .build(), stats: Stats::default(), }), filers: None, + sse_s3: false, } } + /// Whether `bucket` declares SSE-S3 for the files its clients write, as + /// the S3 write path decides it from the table bucket's record: a Lance + /// bucket with `sseAlgorithm` AES256. Read from the filer each time, so a + /// bucket recreated with another setting is written as it now declares. + pub async fn bucket_encrypts(&self, bucket: &str) -> anyhow::Result { + let entry = self + .shared + .source + .lookup(&self.shared.buckets_dir, bucket) + .await + .with_context(|| format!("looking up bucket {bucket}"))?; + let Some(record) = + entry.and_then(|entry| entry.extended.get(TABLE_BUCKET_METADATA).cloned()) + else { + return Ok(false); + }; + let record: serde_json::Value = serde_json::from_slice(&record) + .with_context(|| format!("bucket {bucket} has an unreadable S3 Tables record"))?; + Ok(record["format"] == "LANCE" + && record["encryptionConfiguration"]["sseAlgorithm"] == "AES256") + } + + /// The same storage, writing objects encrypted with SSE-S3 when `sse_s3`, + /// as the bucket being written asks. + pub fn encrypting(&self, sse_s3: bool) -> Self { + Self { + shared: self.shared.clone(), + filers: self.filers.clone(), + sse_s3, + } + } + + /// Removes the table at `location` (`s3://bucket/path`) with everything + /// under it. + pub async fn remove_table(&self, location: &str) -> anyhow::Result<()> { + let rest = location + .strip_prefix("s3://") + .with_context(|| format!("{location} is not an s3:// location"))? + .trim_end_matches('/'); + let (bucket, path) = rest + .split_once('/') + .with_context(|| format!("{location} names no table under its bucket"))?; + if !plain_name(bucket) || !path.split('/').all(plain_name) { + bail!("{location} is not a table path"); + } + let (parent, name) = match path.rsplit_once('/') { + Some((parent, name)) => ( + format!("{}/{bucket}/{parent}", self.shared.buckets_dir), + name, + ), + None => (format!("{}/{bucket}", self.shared.buckets_dir), path), + }; + self.shared.stats.deletes.fetch_add(1, Ordering::Relaxed); + self.shared + .source + .delete(&parent, name, true) + .await + .inspect_err(|_| { + self.shared.stats.failures.fetch_add(1, Ordering::Relaxed); + })?; + let under = format!("{parent}/{name}/"); + self.shared + .objects + .invalidate_entries_if(move |key, _| key.starts_with(&under)) + .map_err(|err| anyhow::anyhow!("forgetting {location}: {err}"))?; + Ok(()) + } + /// The filers as the masters now list them. pub fn set_filers(&self, addresses: &[String]) -> anyhow::Result<()> { match &self.filers { @@ -417,23 +663,28 @@ impl DirectReads { fn report_every(&self, period: Duration) { let shared = Arc::downgrade(&self.shared); tokio::spawn(async move { - let mut last = (0, 0, 0); + let mut last = (0, 0, 0, 0); loop { tokio::time::sleep(period).await; let Some(s) = shared.upgrade() else { return }; let now = ( s.stats.reads.load(Ordering::Relaxed), + s.stats.writes.load(Ordering::Relaxed) + + s.stats.deletes.load(Ordering::Relaxed), s.stats.lookups.load(Ordering::Relaxed), s.stats.failures.load(Ordering::Relaxed), ); if now != last { tracing::info!( - "direct reads: {} reads ({} MiB), {} lookups, {} listings, {} failures", + "direct storage: {} reads ({} MiB), {} lookups, {} listings, {} writes ({} MiB), {} deletes, {} failures", now.0, s.stats.bytes.load(Ordering::Relaxed) >> 20, - now.1, + now.2, s.stats.listings.load(Ordering::Relaxed), - now.2 + s.stats.writes.load(Ordering::Relaxed), + s.stats.written.load(Ordering::Relaxed) >> 20, + s.stats.deletes.load(Ordering::Relaxed), + now.3 ); last = now; } @@ -449,6 +700,7 @@ impl WrappingObjectStore for DirectReads { inner: original, bucket: bucket.into(), reads: self.shared.clone(), + sse_s3: self.sse_s3, }), _ => original, } @@ -589,6 +841,14 @@ fn generic(err: anyhow::Error) -> Error { } } +/// Lance commits with a conditional put and never copies or renames. +fn not_implemented(operation: &str) -> Error { + Error::NotImplemented { + operation: operation.to_string(), + implementer: "DirectStorage".to_string(), + } +} + fn not_found(location: &Path) -> Error { Error::NotFound { path: location.to_string(), @@ -596,11 +856,12 @@ fn not_found(location: &Path) -> Error { } } -#[derive(Debug)] +#[derive(Debug, Clone)] struct DirectStore { inner: Arc, bucket: Arc, reads: Arc, + sse_s3: bool, } impl fmt::Display for DirectStore { @@ -626,6 +887,16 @@ impl Object { version: None, } } + + /// What a read reports besides the bytes: the entry's content type. + fn attributes(&self) -> Attributes { + match self.entry.attributes.as_ref().map(|a| a.mime.as_str()) { + Some(mime) if !mime.is_empty() => { + Attributes::from_iter([(Attribute::ContentType, mime.to_string())]) + } + _ => Attributes::new(), + } + } } impl DirectStore { @@ -781,16 +1052,30 @@ impl DirectStore { Ok(objects) } + /// Removes the object at `location`; a missing one is no error. + async fn remove(&self, location: &Path) -> Result<()> { + let path = self.filer_path(location); + let (dir, name) = path.rsplit_once('/').expect("paths are absolute"); + self.reads.stats.deletes.fetch_add(1, Ordering::Relaxed); + self.reads + .source + .delete(dir, name, false) + .await + .inspect_err(|_| { + self.reads.stats.failures.fetch_add(1, Ordering::Relaxed); + }) + .with_context(|| format!("deleting {path}")) + .map_err(generic)?; + self.reads.objects.invalidate(&path); + Ok(()) + } + fn stream( &self, prefix: Option<&Path>, offset: Option<&Path>, ) -> BoxStream<'static, Result> { - let store = DirectStore { - inner: self.inner.clone(), - bucket: self.bucket.clone(), - reads: self.reads.clone(), - }; + let store = self.clone(); let (prefix, offset) = (prefix.cloned(), offset.cloned()); futures::stream::once(async move { store.list_all(prefix, offset).await }) .flat_map(|listed| match listed { @@ -810,7 +1095,25 @@ impl ObjectStore for DirectStore { payload: PutPayload, opts: PutOptions, ) -> Result { - self.inner.put_opts(location, payload, opts).await + let exclusive = match opts.mode { + PutMode::Overwrite => false, + PutMode::Create => true, + PutMode::Update(_) => { + return Err(Error::NotSupported { + source: "the filer keeps no object versions to update against".into(), + }); + } + }; + let upload = Upload::start(self.clone(), location.clone(), &opts.attributes)?; + let data: Bytes = payload.into(); + let md5 = Md5::digest(&data); + if let Err(err) = upload.put(data).await { + if let Err(discard) = upload.discard_all().await { + tracing::warn!("{location}: discarding after a failed upload: {discard:#}"); + } + return Err(err); + } + upload.finish(exclusive, Some(md5.to_vec())).await } async fn put_multipart_opts( @@ -818,7 +1121,11 @@ impl ObjectStore for DirectStore { location: &Path, opts: PutMultipartOptions, ) -> Result> { - self.inner.put_multipart_opts(location, opts).await + Ok(Box::new(Upload::start( + self.clone(), + location.clone(), + &opts.attributes, + )?)) } async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { @@ -835,7 +1142,7 @@ impl ObjectStore for DirectStore { payload: GetResultPayload::Stream(futures::stream::empty().boxed()), meta, range: 0..0, - attributes: Default::default(), + attributes: object.attributes(), }); } let range = match &options.range { @@ -852,7 +1159,7 @@ impl ObjectStore for DirectStore { payload: GetResultPayload::Stream(futures::stream::once(async { Ok(bytes) }).boxed()), meta, range, - attributes: Default::default(), + attributes: object.attributes(), }) } @@ -883,16 +1190,14 @@ impl ObjectStore for DirectStore { &self, locations: BoxStream<'static, Result>, ) -> BoxStream<'static, Result> { - let store = DirectStore { - inner: self.inner.clone(), - bucket: self.bucket.clone(), - reads: self.reads.clone(), - }; - self.inner - .delete_stream(locations) - .inspect(move |deleted| { - if let Ok(location) = deleted { - store.reads.objects.invalidate(&store.filer_path(location)); + let store = self.clone(); + locations + .then(move |location| { + let store = store.clone(); + async move { + let location = location?; + store.remove(&location).await?; + Ok(location) } }) .boxed() @@ -927,12 +1232,363 @@ impl ObjectStore for DirectStore { Ok(result) } - async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { - self.inner.copy_opts(from, to, options).await + async fn copy_opts(&self, _from: &Path, _to: &Path, _options: CopyOptions) -> Result<()> { + Err(not_implemented("copy")) } - async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> { - self.inner.rename_opts(from, to, options).await + async fn rename_opts(&self, _from: &Path, _to: &Path, _options: RenameOptions) -> Result<()> { + Err(not_implemented("rename")) + } +} + +/// An object being written: its chunks go to the volume servers as they +/// arrive, the entry is created when they are all there. +struct Upload { + store: DirectStore, + location: Path, + path: String, + /// The collection the gateway names for the bucket: its filer group's, + /// or none, so the filer's rules for the path pick it. + collection: String, + mime: String, + encoder: Arc, + state: Arc>, +} + +#[derive(Default)] +struct UploadState { + chunks: Vec, + /// Where the next part starts. + offset: u64, + /// The entry was created, or may have been: nothing more is done with + /// this upload, and nothing it uploaded is discarded. + done: bool, +} + +impl fmt::Debug for Upload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Upload").field("path", &self.path).finish() + } +} + +impl Upload { + fn start(store: DirectStore, location: Path, attributes: &Attributes) -> Result { + let path = store.filer_path(&location); + let encoder = EntryEncoder::new(store.reads.keys.clone(), store.sse_s3, store.reads.cipher) + .with_context(|| format!("encoding {path}")) + .map_err(generic)?; + let collection = match store.reads.filer_group.as_str() { + "" => String::new(), + group => format!("{group}_{}", store.bucket), + }; + let mime = attributes + .get(&Attribute::ContentType) + .map_or(OCTET_STREAM, |v| v.as_ref()) + .to_string(); + Ok(Self { + store, + location, + path, + collection, + mime, + encoder: Arc::new(encoder), + state: Arc::default(), + }) + } + + /// Uploads `data` as the chunks following what was put before. + fn put(&self, data: Bytes) -> UploadPart { + let offset = { + let mut state = self.state.lock().unwrap(); + let offset = state.offset; + state.offset += data.len() as u64; + offset + }; + let (store, path, collection, encoder, state) = ( + self.store.clone(), + self.path.clone(), + self.collection.clone(), + self.encoder.clone(), + self.state.clone(), + ); + Box::pin(async move { + let chunk_size = store.reads.chunk_size; + let mut at = offset; + for piece in data.chunks(chunk_size.max(1)) { + let encoded = encoder + .encode(at, piece.to_vec()) + .with_context(|| format!("encoding {path} at {at}")) + .map_err(generic)?; + let size = piece.len() as u64; + let data = Bytes::from(encoded.data); + let source = &store.reads.source; + let failed = |err: anyhow::Error, what: &str| { + store.reads.stats.failures.fetch_add(1, Ordering::Relaxed); + generic(err.context(format!("{what} {path} at {at}"))) + }; + let reserved = Reserved { + source: source.clone(), + path: path.clone(), + assignment: Some( + source + .assign(&path, &collection, data.len() as u64) + .await + .map_err(|err| failed(err, "assigning"))?, + ), + }; + let mut chunk = match source.write(reserved.assignment(), &path, at, data).await { + Ok(chunk) => chunk, + Err(err) => { + reserved.discard().await; + return Err(failed(err, "uploading")); + } + }; + chunk.size = size; + chunk.cipher_key = encoded.cipher_key; + chunk.sse_type = encoded.sse_type; + chunk.sse_metadata = encoded.sse_metadata; + store.reads.stats.written.fetch_add(size, Ordering::Relaxed); + state.lock().unwrap().chunks.push(Uploaded { + chunk, + assignment: reserved.disarm(), + }); + at += size; + } + Ok(()) + }) + } + + /// Creates the entry over the chunks uploaded so far. `md5` is the whole + /// object's, known for a one-shot put; a multipart object's ETag derives + /// from its chunks, as through the gateway. + async fn finish(&self, exclusive: bool, md5: Option>) -> Result { + let mut chunks = { + let mut state = self.state.lock().unwrap(); + if state.done { + return Err(generic(anyhow::anyhow!( + "{} was already completed, or its completion is in doubt", + self.path + ))); + } + state.done = true; + std::mem::take(&mut state.chunks) + }; + chunks.sort_by_key(|u| u.chunk.offset); + let size = chunks + .iter() + .map(|u| u.chunk.offset as u64 + u.chunk.size) + .max() + .unwrap_or(0); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) as i64; + let (dir, name) = self.path.rsplit_once('/').expect("paths are absolute"); + let mut entry = Entry { + name: name.to_string(), + attributes: Some(FuseAttributes { + crtime: now, + mtime: now, + file_mode: FILE_MODE, + mime: self.mime.clone(), + file_size: size, + md5: md5.clone().unwrap_or_default(), + ..Default::default() + }), + chunks: chunks.iter().map(|u| u.chunk.clone()).collect(), + ..Default::default() + }; + if let Some(md5) = &md5 { + entry + .extended + .insert(ETAG_ATTRIBUTE.to_string(), hex::encode(md5).into_bytes()); + } + let attributes = self.encoder.attributes().map_err(generic)?; + entry.extended.extend(attributes); + let e_tag = etag(&entry); + let created = self.store.reads.source.create(dir, entry, exclusive).await; + if let Err(err) = created { + // A create the filer took and never answered may still be applied, + // and a retried exclusive create may find this very upload in + // place: chunks the entry references, or may yet reference, are + // kept. Only a create the filer refused has its chunks discarded. + let saved = match self.store.reads.source.lookup(dir, name).await { + Ok(Some(saved)) => saved + .chunks + .first() + .is_some_and(|c| chunks.iter().any(|u| u.chunk.file_id == c.file_id)), + Ok(None) => false, + Err(lookup) => { + tracing::warn!("{}: keeping its chunks: {lookup:#}", self.path); + self.store + .reads + .stats + .failures + .fetch_add(1, Ordering::Relaxed); + return Err(generic(err.context(format!("creating {}", self.path)))); + } + }; + if saved { + tracing::warn!("{}: created despite {err:#}", self.path); + } else if outcome_uncertain(&err) { + tracing::warn!("{}: keeping its chunks: {err:#}", self.path); + self.store + .reads + .stats + .failures + .fetch_add(1, Ordering::Relaxed); + return Err(generic(err.context(format!("creating {}", self.path)))); + } else { + for uploaded in &chunks { + if let Err(discard) = + self.store.reads.source.discard(&uploaded.assignment).await + { + tracing::warn!( + "{}: discarding {}: {discard:#}", + self.path, + uploaded.chunk.file_id + ); + } + } + self.store + .reads + .stats + .failures + .fetch_add(1, Ordering::Relaxed); + return Err(if sw_client::is_entry_exists(&err) { + Error::AlreadyExists { + path: self.location.to_string(), + source: format!("{err:#}").into(), + } + } else { + generic(err.context(format!("creating {}", self.path))) + }); + } + } + self.store + .reads + .stats + .writes + .fetch_add(1, Ordering::Relaxed); + self.store.reads.objects.invalidate(&self.path); + Ok(PutResult { + e_tag, + version: None, + }) + } + + /// Discards every chunk uploaded; those a volume server would not give up + /// are kept for another attempt, by a later abort or the drop. + async fn discard_all(&self) -> Result<()> { + let chunks = { + let mut state = self.state.lock().unwrap(); + state.done = true; + std::mem::take(&mut state.chunks) + }; + let kept = discard(&*self.store.reads.source, &self.path, chunks).await; + if kept.is_empty() { + return Ok(()); + } + let ids: Vec<_> = kept.iter().map(|u| u.chunk.file_id.as_str()).collect(); + let err = anyhow::anyhow!("{}: still holding {}", self.path, ids.join(", ")); + self.state.lock().unwrap().chunks.extend(kept); + Err(generic(err)) + } +} + +/// Discards `chunks` on the volume servers and returns those still there. +async fn discard(source: &dyn Source, path: &str, chunks: Vec) -> Vec { + let mut kept = Vec::new(); + for uploaded in chunks { + if let Err(err) = source.discard(&uploaded.assignment).await { + tracing::warn!( + "{path}: discarding {}: {err:#}", + uploaded.assignment.file_id + ); + kept.push(uploaded); + } + } + kept +} + +/// A needle assigned on a volume server while its write is in flight. The +/// volume server may hold it although the write's answer never came, and +/// nothing else will ever name it: a write abandoned, as when the request +/// is cancelled, or failed gives the needle back. +struct Reserved { + source: Arc, + path: String, + assignment: Option, +} + +impl Reserved { + fn assignment(&self) -> &Assignment { + self.assignment.as_ref().expect("reserved until disarmed") + } + + /// The write succeeded: the entry will name the needle. + fn disarm(mut self) -> Assignment { + self.assignment.take().expect("reserved until disarmed") + } + + async fn discard(mut self) { + if let Some(assignment) = self.assignment.take() { + discard_needle(&*self.source, &self.path, &assignment).await; + } + } +} + +impl Drop for Reserved { + fn drop(&mut self) { + let Some(assignment) = self.assignment.take() else { + return; + }; + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let (source, path) = (self.source.clone(), self.path.clone()); + runtime.spawn(async move { + discard_needle(&*source, &path, &assignment).await; + }); + } +} + +async fn discard_needle(source: &dyn Source, path: &str, assignment: &Assignment) { + if let Err(err) = source.discard(assignment).await { + tracing::warn!("{path}: discarding {}: {err:#}", assignment.file_id); + } +} + +/// An upload dropped before it was completed or aborted, as when the request +/// writing it was cancelled, discards what it uploaded; one completed, or +/// whose completion is in doubt, has nothing left to discard. +impl Drop for Upload { + fn drop(&mut self) { + let chunks = std::mem::take(&mut self.state.lock().unwrap().chunks); + if chunks.is_empty() { + return; + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let (source, path) = (self.store.reads.source.clone(), self.path.clone()); + runtime.spawn(async move { + discard(&*source, &path, chunks).await; + }); + } +} + +#[async_trait::async_trait] +impl MultipartUpload for Upload { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + self.put(data.into()) + } + + async fn complete(&mut self) -> Result { + self.finish(false, None).await + } + + async fn abort(&mut self) -> Result<()> { + self.discard_all().await } } @@ -976,13 +1632,33 @@ impl VolumeTransport for JwtTransport { async fn write( &self, - _location: &str, - _file_id: &str, - _auth: &str, - _filename: &str, - _data: Bytes, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, ) -> anyhow::Result<()> { - bail!("direct reads never write") + self.http + .write(location, file_id, auth, filename, data) + .await + } + + async fn write_synced( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + fsync: bool, + ) -> anyhow::Result { + self.http + .write_synced(location, file_id, auth, filename, data, fsync) + .await + } + + async fn delete(&self, location: &str, file_id: &str, auth: &str) -> anyhow::Result<()> { + self.http.delete(location, file_id, auth).await } fn name(&self) -> &'static str { @@ -1005,15 +1681,39 @@ mod tests { struct Fake { entries: Mutex>, bytes: Mutex>>, + /// Needles uploaded through [`Source::upload`], by file id. + needles: Mutex>>, lookups: AtomicU64, cached: AtomicU64, + uploads: AtomicU64, + discarded: AtomicU64, fail_reads: bool, + fail_lookups: bool, + /// Creates save the entry and then answer with an error, as a filer + /// whose reply was lost. + lose_create_replies: bool, + /// Creates never answer and never save, as a filer that took the call + /// and went silent. + silent_creates: bool, + /// Assignments after this many fail. + uploads_allowed: Option, + /// Chunk writes never finish, as to a volume server that stopped + /// answering. + stall_writes: bool, + fail_deletes: bool, + /// This many discards fail before any succeeds. + failing_discards: AtomicU64, + /// The collections uploads named. + collections: Mutex>, } #[async_trait::async_trait] impl Source for Fake { async fn lookup(&self, dir: &str, name: &str) -> anyhow::Result> { self.lookups.fetch_add(1, Ordering::Relaxed); + if self.fail_lookups { + bail!("the filer is unreachable"); + } Ok(self .entries .lock() @@ -1065,13 +1765,115 @@ mod tests { if self.fail_reads { bail!("volume server unreachable"); } - let bytes = self.bytes.lock().unwrap()[&object.entry.name].clone(); + let bytes = match self.bytes.lock().unwrap().get(&object.entry.name) { + Some(bytes) => bytes.clone(), + // Written through the store: the needles, decoded as a volume + // read would be. + None => { + let needles = self.needles.lock().unwrap(); + let mut whole = vec![0u8; object.size as usize]; + for chunk in &object.entry.chunks { + let raw = needles[&chunk.file_id].clone(); + let plain = object.decoder.decode(chunk, 0, raw)?; + let at = chunk.offset as usize; + whole[at..at + plain.len()].copy_from_slice(&plain); + } + whole + } + }; let end = ((offset + len) as usize).min(bytes.len()); let mut out = vec![0u8; len as usize]; out[..end - offset as usize].copy_from_slice(&bytes[offset as usize..end]); Ok(out) } + async fn assign( + &self, + _path: &str, + collection: &str, + _size: u64, + ) -> anyhow::Result { + self.collections + .lock() + .unwrap() + .insert(collection.to_string()); + let n = self.uploads.fetch_add(1, Ordering::Relaxed) + 1; + if self.uploads_allowed.is_some_and(|allowed| n > allowed) { + bail!("volume server unreachable"); + } + Ok(Assignment { + file_id: format!("7,{n:08x}"), + url: "volume:8080".into(), + auth: String::new(), + fsync: false, + }) + } + + async fn write( + &self, + assignment: &Assignment, + _path: &str, + offset: u64, + data: Bytes, + ) -> anyhow::Result { + if self.stall_writes { + std::future::pending::<()>().await; + } + self.needles + .lock() + .unwrap() + .insert(assignment.file_id.clone(), data.to_vec()); + Ok(FileChunk { + file_id: assignment.file_id.clone(), + offset: offset as i64, + size: data.len() as u64, + e_tag: BASE64_STANDARD.encode(Md5::digest(&data)), + ..Default::default() + }) + } + + async fn create(&self, dir: &str, entry: Entry, exclusive: bool) -> anyhow::Result<()> { + if self.silent_creates { + return Err( + anyhow::anyhow!("filer took the create and went silent").context(NoAnswer) + ); + } + let mut entries = self.entries.lock().unwrap(); + let path = format!("{dir}/{}", entry.name); + if exclusive && entries.contains_key(&path) { + return Err(anyhow::anyhow!("CreateEntry: {path}: entry already exists") + .context(sw_client::EntryExists)); + } + entries.insert(path, entry); + if self.lose_create_replies { + bail!("the filer did not answer"); + } + Ok(()) + } + + async fn delete(&self, dir: &str, name: &str, recursive: bool) -> anyhow::Result<()> { + if self.fail_deletes { + bail!("the filer is unreachable"); + } + let path = format!("{dir}/{name}"); + let under = format!("{path}/"); + self.entries + .lock() + .unwrap() + .retain(|key, _| key != &path && !(recursive && key.starts_with(&under))); + Ok(()) + } + + async fn discard(&self, assignment: &Assignment) -> anyhow::Result<()> { + if self.failing_discards.load(Ordering::Relaxed) > 0 { + self.failing_discards.fetch_sub(1, Ordering::Relaxed); + bail!("volume server unreachable"); + } + self.discarded.fetch_add(1, Ordering::Relaxed); + self.needles.lock().unwrap().remove(&assignment.file_id); + Ok(()) + } + fn name(&self) -> String { "fake".into() } @@ -1171,6 +1973,7 @@ mod tests { inner: inner.clone(), bucket: "vb".into(), reads: reads.shared.clone(), + sse_s3: false, }; assert_eq!( store.filer_path(&Path::from("vector/idx/data/a.lance")), @@ -1635,7 +2438,12 @@ mod tests { let (store, _) = store(fake.clone()).await; ranges(&store, DATA, &[0..5]).await; store.delete(&Path::from(DATA)).await.unwrap(); - ranges(&store, DATA, &[0..5]).await; + // Written again at the same path: the next read looks it up afresh. + fake.add(&format!("/buckets/vb/{DATA}"), b"hello again", |_| {}); + assert_eq!( + ranges(&store, DATA, &[6..11]).await[0], + Bytes::from_static(b"again") + ); assert_eq!(fake.lookups.load(Ordering::Relaxed), 2); } @@ -1661,11 +2469,14 @@ mod tests { #[tokio::test] async fn filers_follow_the_masters_and_unreachable_ones_are_passed_over() { + let transport: Arc = Arc::new(HttpTransport::new()); let filers = Filers { clients: RwLock::new(Vec::new()), dial: Arc::new(|addr| FilerClient::connect_lazy_with(addr, None)), current: AtomicUsize::new(0), - volumes: VolumeReader::with_transport_capped(Arc::new(HttpTransport::new()), 0), + volumes: VolumeReader::with_transport_capped(transport.clone(), 0), + writer: Writer::with_transport(transport.clone()), + transport, }; let err = filers.lookup("/buckets", "vb").await.unwrap_err(); assert!(err.to_string().contains("no filer"), "{err:#}"); @@ -1684,4 +2495,643 @@ mod tests { filers.set_addresses(&[]).unwrap(); assert!(filers.lookup("/buckets", "vb").await.is_err()); } + + /// A filer that never answered may still apply the request, which the + /// error says even when the next filer refuses it. + #[tokio::test(start_paused = true)] + async fn filers_that_never_answer_leave_the_outcome_open() { + let transport: Arc = Arc::new(HttpTransport::new()); + let filers = Filers { + clients: RwLock::new(Vec::new()), + dial: Arc::new(|addr| FilerClient::connect_lazy_with(addr, None)), + current: AtomicUsize::new(0), + volumes: VolumeReader::with_transport_capped(transport.clone(), 0), + writer: Writer::with_transport(transport.clone()), + transport, + }; + filers + .set_addresses(&["127.0.0.1:1".into(), "127.0.0.1:2".into()]) + .unwrap(); + let silent = |c: FilerClient| -> BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async move { + if c.addr() == "127.0.0.1:1" { + std::future::pending::<()>().await; + } + bail!("refused") + }) + }; + let err = filers.with("create", silent).await.unwrap_err(); + assert!(outcome_uncertain(&err), "{err:#}"); + assert!(format!("{err:#}").contains("refused"), "{err:#}"); + + let fell_away = |c: FilerClient| -> BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async move { + if c.addr() == "127.0.0.1:1" { + return Err(tonic::Status::unavailable("connection reset").into()); + } + bail!("refused") + }) + }; + let err = filers.with("create", fell_away).await.unwrap_err(); + assert!(outcome_uncertain(&err), "{err:#}"); + assert!(format!("{err:#}").contains("refused"), "{err:#}"); + + let answered = |_: FilerClient| -> BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async { bail!("refused") }) + }; + let err = filers.with("create", answered).await.unwrap_err(); + assert!(!outcome_uncertain(&err), "{err:#}"); + } + + #[test] + fn only_the_filers_missing_entry_error_is_not_found() { + let missing = anyhow::anyhow!("filer: no entry is found in filer store"); + assert!(is_not_found(&missing.context("ListEntries"))); + let remote = anyhow::anyhow!("resolve remote storage client for /buckets/vb/t: not found"); + assert!(!is_not_found(&remote.context("DeleteEntry"))); + } + + /// Failed uploads and deletes show in the storage report. + #[tokio::test] + async fn failed_writes_and_deletes_are_counted() { + let fake = Arc::new(Fake { + uploads_allowed: Some(0), + fail_deletes: true, + ..Default::default() + }); + let reads = DirectReads::new(fake.clone(), "", no_keys()); + let store = reads.wrap("s3$vb", Arc::new(InMemory::new())); + let failures = || reads.shared.stats.failures.load(Ordering::Relaxed); + store + .put( + &Path::from("vector/idx/data/a.lance"), + Bytes::from_static(b"x").into(), + ) + .await + .unwrap_err(); + assert_eq!(failures(), 1); + store + .delete(&Path::from("vector/idx/data/b.lance")) + .await + .unwrap_err(); + assert_eq!(failures(), 2); + reads.remove_table("s3://vb/vector/idx").await.unwrap_err(); + assert_eq!(failures(), 3); + } + + /// A store writing through `fake`, with `chunk_size` bytes per chunk. + fn writing_store( + fake: Arc, + keys: Arc, + chunk_size: usize, + cipher: bool, + sse_s3: bool, + ) -> Arc { + let mut reads = DirectReads::new(fake, "", keys); + { + let shared = Arc::get_mut(&mut reads.shared).unwrap(); + shared.chunk_size = chunk_size; + shared.cipher = cipher; + } + reads + .encrypting(sse_s3) + .wrap("s3$vb", Arc::new(InMemory::new())) + } + + fn entry_of(fake: &Fake, path: &str) -> Entry { + fake.entries.lock().unwrap()[&format!("/buckets/vb/{path}")].clone() + } + + /// A put creates the entry the gateway would, readable back through the + /// store; a conditional create of an existing object is a conflict that + /// leaves no needle behind. + #[tokio::test] + async fn writes_create_the_entry_the_gateway_would() { + let fake = Arc::new(Fake::default()); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let path = Path::from("vector/idx/_versions/2.manifest"); + let body = Bytes::from(vec![9u8; 10_000]); + let result = store + .put_opts(&path, body.clone().into(), PutMode::Create.into()) + .await + .unwrap(); + let md5 = hex::encode(Md5::digest(&body)); + assert_eq!(result.e_tag.as_deref(), Some(format!("\"{md5}\"").as_str())); + + let entry = entry_of(&fake, "vector/idx/_versions/2.manifest"); + let attributes = entry.attributes.as_ref().unwrap(); + assert_eq!(attributes.file_size, 10_000); + assert_eq!(hex::encode(&attributes.md5), md5); + assert_eq!(attributes.mime, OCTET_STREAM); + assert_eq!(entry.extended[ETAG_ATTRIBUTE], md5.as_bytes()); + assert_eq!(entry.chunks.len(), 3, "4096-byte chunks"); + assert_eq!(entry.chunks[2].offset, 8192); + assert!( + entry + .chunks + .iter() + .all(|c| c.cipher_key.is_empty() && c.sse_type == 0) + ); + + let got = store.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(got, body); + let meta = store.head(&path).await.unwrap(); + assert_eq!(meta.size, 10_000); + assert_eq!(meta.e_tag, result.e_tag); + + let conflict = store + .put_opts( + &path, + Bytes::from_static(b"again").into(), + PutMode::Create.into(), + ) + .await + .unwrap_err(); + assert!( + matches!(conflict, Error::AlreadyExists { .. }), + "{conflict}" + ); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 1); + assert_eq!(fake.needles.lock().unwrap().len(), 3); + + store + .put_opts( + &path, + Bytes::from_static(b"again").into(), + PutMode::Overwrite.into(), + ) + .await + .unwrap(); + let got = store.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(got, Bytes::from_static(b"again")); + } + + /// A multipart upload's parts become chunks in order, its ETag derives + /// from them as the gateway's does, and an abort leaves no needle behind. + #[tokio::test] + async fn multipart_uploads_become_chunks_in_order() { + let fake = Arc::new(Fake::default()); + let store = writing_store(fake.clone(), no_keys(), 1 << 20, false, false); + let path = Path::from("vector/idx/data/abcd.lance"); + let mut upload = store.put_multipart(&path).await.unwrap(); + let parts = [vec![1u8; 3000], vec![2u8; 3000], vec![3u8; 1000]]; + for part in &parts { + upload + .put_part(Bytes::from(part.clone()).into()) + .await + .unwrap(); + } + let result = upload.complete().await.unwrap(); + let entry = entry_of(&fake, "vector/idx/data/abcd.lance"); + assert_eq!(entry.chunks.len(), 3); + assert_eq!(entry.chunks[1].offset, 3000); + assert!(entry.attributes.as_ref().unwrap().md5.is_empty()); + assert!(!entry.extended.contains_key(ETAG_ATTRIBUTE)); + let digests: Vec> = parts.iter().map(|p| Md5::digest(p).to_vec()).collect(); + let want = format!("\"{}-3\"", hex::encode(Md5::digest(digests.concat()))); + assert_eq!(result.e_tag.as_deref(), Some(want.as_str())); + let got = store.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(got, Bytes::from(parts.concat())); + + let mut aborted = store + .put_multipart(&Path::from("vector/idx/data/gone.lance")) + .await + .unwrap(); + aborted + .put_part(Bytes::from_static(b"never").into()) + .await + .unwrap(); + aborted.abort().await.unwrap(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 1); + assert!( + fake.entries + .lock() + .unwrap() + .keys() + .all(|k| !k.ends_with("gone.lance")) + ); + } + + /// With SSE-S3 the needles hold ciphertext under a per-object key the + /// entry carries, and with the cluster's volume encryption a per-chunk + /// key; both read back through the store, and a chunk is never plain. + #[tokio::test] + async fn encrypted_writes_read_back() { + let keys = Arc::new( + SseS3Keys::load( + &SseConfig { + key: Some("secret".into()), + ..Default::default() + }, + None, + ) + .unwrap(), + ); + for (cipher, sse_s3) in [(false, true), (true, false), (true, true)] { + let fake = Arc::new(Fake::default()); + let store = writing_store(fake.clone(), keys.clone(), 4096, cipher, sse_s3); + let path = Path::from("vector/idx/_versions/3.manifest"); + let body: Vec = (0..10_000u32).map(|i| (i % 251) as u8).collect(); + store + .put(&path, Bytes::from(body.clone()).into()) + .await + .unwrap(); + let entry = entry_of(&fake, "vector/idx/_versions/3.manifest"); + for chunk in &entry.chunks { + let needle = &fake.needles.lock().unwrap()[&chunk.file_id]; + let plain = + &body[chunk.offset as usize..chunk.offset as usize + chunk.size as usize]; + assert_ne!(needle, plain, "cipher {cipher} sse {sse_s3}"); + assert_eq!(!chunk.cipher_key.is_empty(), cipher); + assert_eq!(chunk.sse_type == SSE_S3_TYPE, sse_s3); + } + assert_eq!( + entry.extended.contains_key("X-Amz-Server-Side-Encryption"), + sse_s3 + ); + if sse_s3 { + // No two chunks share a keystream; the wrapped keys always + // differ, so the IVs are what is compared. + let ivs: std::collections::HashSet = entry + .chunks + .iter() + .map(|c| { + let meta: serde_json::Value = + serde_json::from_slice(&c.sse_metadata).unwrap(); + meta["iv"].as_str().unwrap().to_string() + }) + .collect(); + assert_eq!(ivs.len(), entry.chunks.len()); + } + let got = store.get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!( + got, + Bytes::from(body.clone()), + "cipher {cipher} sse {sse_s3}" + ); + + // Parts of a multipart upload start wherever the previous one + // ended, inside a cipher block included. + let parts = Path::from("vector/idx/data/parts.lance"); + let mut upload = store.put_multipart(&parts).await.unwrap(); + for part in body.chunks(3000) { + upload + .put_part(Bytes::from(part.to_vec()).into()) + .await + .unwrap(); + } + upload.complete().await.unwrap(); + let got = store.get(&parts).await.unwrap().bytes().await.unwrap(); + assert_eq!( + got, + Bytes::from(body), + "unaligned parts, cipher {cipher} sse {sse_s3}" + ); + } + } + + /// An SSE-S3 bucket cannot be written without the KEK. + #[tokio::test] + async fn sse_s3_writes_need_the_kek() { + let store = writing_store(Arc::new(Fake::default()), no_keys(), 4096, false, true); + let err = store + .put(&Path::from("vector/idx/x"), Bytes::from_static(b"x").into()) + .await + .unwrap_err(); + assert!(err.to_string().contains("s3.sse.kek"), "{err}"); + } + + /// Deletes remove the entry and forget the cached object; a table is + /// removed with everything under it. + #[tokio::test] + async fn deletes_remove_entries_and_tables() { + let fake = Arc::new(Fake::default()); + let mut reads = DirectReads::new(fake.clone(), "", no_keys()); + Arc::get_mut(&mut reads.shared).unwrap().chunk_size = 4096; + let store = reads.wrap("s3$vb", Arc::new(InMemory::new())); + for name in [ + "vector/idx/data/a.lance", + "vector/idx/data/b.lance", + "vector/idx/_versions/1.manifest", + ] { + store + .put(&Path::from(name), Bytes::from_static(b"bytes").into()) + .await + .unwrap(); + } + for name in ["vector/idx/data/a.lance", "vector/idx/data/b.lance"] { + store.get(&Path::from(name)).await.unwrap(); + } + store + .delete(&Path::from("vector/idx/data/a.lance")) + .await + .unwrap(); + let err = store + .get(&Path::from("vector/idx/data/a.lance")) + .await + .unwrap_err(); + assert!(matches!(err, Error::NotFound { .. }), "{err}"); + store + .delete(&Path::from("vector/idx/data/a.lance")) + .await + .unwrap(); + + reads.remove_table("s3://vb/vector/idx").await.unwrap(); + assert!(fake.entries.lock().unwrap().is_empty()); + for name in ["vector/idx/_versions/1.manifest", "vector/idx/data/b.lance"] { + let err = store.get(&Path::from(name)).await.unwrap_err(); + assert!(matches!(err, Error::NotFound { .. }), "{name}: {err}"); + } + } + + /// A create the filer saved but did not answer keeps its chunks and + /// counts as written; a put that fails midway leaves no needle behind. + #[tokio::test] + async fn uncertain_and_failed_writes_leave_nothing_wrong_behind() { + let fake = Arc::new(Fake { + lose_create_replies: true, + ..Default::default() + }); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let path = Path::from("vector/idx/_versions/5.manifest"); + store + .put_opts( + &path, + Bytes::from(vec![1u8; 5000]).into(), + PutMode::Create.into(), + ) + .await + .unwrap(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 0); + assert_eq!(fake.needles.lock().unwrap().len(), 2); + assert_eq!( + entry_of(&fake, "vector/idx/_versions/5.manifest") + .chunks + .len(), + 2 + ); + + // Nor are they discarded when the filer cannot be asked. + let fake = Arc::new(Fake { + lose_create_replies: true, + fail_lookups: true, + ..Default::default() + }); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + store + .put_opts( + &path, + Bytes::from(vec![1u8; 5000]).into(), + PutMode::Create.into(), + ) + .await + .unwrap_err(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 0); + assert_eq!(fake.needles.lock().unwrap().len(), 2); + + let fake = Arc::new(Fake { + uploads_allowed: Some(1), + ..Default::default() + }); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + store + .put( + &Path::from("vector/idx/data/half.lance"), + Bytes::from(vec![2u8; 5000]).into(), + ) + .await + .unwrap_err(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 1); + assert!(fake.needles.lock().unwrap().is_empty()); + assert!(fake.entries.lock().unwrap().is_empty()); + } + + /// A create the filer took and never answered keeps its chunks, since + /// it may still be applied; the upload can then be neither completed + /// again nor aborted into discarding them. + #[tokio::test] + async fn an_unanswered_create_keeps_its_chunks() { + let fake = Arc::new(Fake { + silent_creates: true, + ..Default::default() + }); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let path = Path::from("vector/idx/data/silent.lance"); + let mut upload = store.put_multipart(&path).await.unwrap(); + upload + .put_part(Bytes::from(vec![3u8; 5000]).into()) + .await + .unwrap(); + upload.complete().await.unwrap_err(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 0); + assert_eq!(fake.needles.lock().unwrap().len(), 2); + upload.complete().await.unwrap_err(); + upload.abort().await.unwrap(); + drop(upload); + tokio::task::yield_now().await; + assert_eq!( + fake.discarded.load(Ordering::Relaxed), + 0, + "kept through abort and drop" + ); + } + + /// An upload dropped unfinished, as by a cancelled request, discards + /// what it uploaded. + #[tokio::test] + async fn a_dropped_upload_discards_its_chunks() { + let fake = Arc::new(Fake::default()); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let mut upload = store + .put_multipart(&Path::from("vector/idx/data/gone.lance")) + .await + .unwrap(); + upload + .put_part(Bytes::from(vec![4u8; 5000]).into()) + .await + .unwrap(); + drop(upload); + for _ in 0..50 { + if fake.discarded.load(Ordering::Relaxed) == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(fake.discarded.load(Ordering::Relaxed), 2); + } + + /// An abort that cannot discard every chunk keeps the ones left for a + /// later attempt, and the upload cannot be completed any more. + #[tokio::test] + async fn an_abort_keeps_what_it_could_not_discard() { + let fake = Arc::new(Fake { + failing_discards: AtomicU64::new(1), + ..Default::default() + }); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let mut upload = store + .put_multipart(&Path::from("vector/idx/data/kept.lance")) + .await + .unwrap(); + upload + .put_part(Bytes::from(vec![5u8; 10_000]).into()) + .await + .unwrap(); + upload.abort().await.unwrap_err(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 2); + upload.complete().await.unwrap_err(); + upload.abort().await.unwrap(); + assert_eq!(fake.discarded.load(Ordering::Relaxed), 3); + assert!(fake.needles.lock().unwrap().is_empty()); + } + + /// A chunk write cancelled in flight, as when the request writing it is + /// dropped, gives its assigned needle back. + #[tokio::test] + async fn a_cancelled_chunk_write_gives_its_needle_back() { + let fake = Arc::new(Fake { + stall_writes: true, + ..Default::default() + }); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let put = tokio::spawn(async move { + store + .put( + &Path::from("vector/idx/data/cut.lance"), + Bytes::from(vec![6u8; 100]).into(), + ) + .await + }); + for _ in 0..100 { + if fake.uploads.load(Ordering::Relaxed) == 1 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(fake.uploads.load(Ordering::Relaxed), 1, "assigned"); + put.abort(); + put.await.unwrap_err(); + for _ in 0..100 { + if fake.discarded.load(Ordering::Relaxed) == 1 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(fake.discarded.load(Ordering::Relaxed), 1); + } + + /// Chunks name the collection the gateway names: the filer group's for + /// the bucket, or none, leaving it to the filer's rules for the path. + #[tokio::test] + async fn chunks_go_to_the_buckets_collection() { + for (group, want) in [("", ""), ("eu", "eu_vb")] { + let fake = Arc::new(Fake::default()); + let mut reads = DirectReads::new(fake.clone(), "", no_keys()); + { + let shared = Arc::get_mut(&mut reads.shared).unwrap(); + shared.chunk_size = 4096; + shared.filer_group = group.to_string(); + } + let store = reads.wrap("s3$vb", Arc::new(InMemory::new())); + store + .put(&Path::from("vector/idx/x"), Bytes::from_static(b"x").into()) + .await + .unwrap(); + let collections = fake.collections.lock().unwrap(); + assert_eq!( + collections.iter().collect::>(), + vec![&want.to_string()] + ); + } + } + + /// The content type a put carries becomes the entry's mime, as it does + /// through the gateway. + #[tokio::test] + async fn a_put_keeps_its_content_type() { + let fake = Arc::new(Fake::default()); + let store = writing_store(fake.clone(), no_keys(), 4096, false, false); + let mut attributes = Attributes::new(); + attributes.insert(Attribute::ContentType, "application/json".into()); + let opts = PutOptions { + attributes, + ..Default::default() + }; + store + .put_opts( + &Path::from("vector/idx/typed"), + Bytes::from_static(b"{}").into(), + opts, + ) + .await + .unwrap(); + let typed = entry_of(&fake, "vector/idx/typed"); + assert_eq!(typed.attributes.unwrap().mime, "application/json"); + let read = store.get(&Path::from("vector/idx/typed")).await.unwrap(); + assert_eq!( + read.attributes + .get(&Attribute::ContentType) + .map(|v| v.as_ref()), + Some("application/json") + ); + } + + /// A bucket encrypts when its S3 Tables record says LANCE and AES256, + /// is read afresh each time, and refuses a record that cannot be read. + #[tokio::test] + async fn bucket_encryption_follows_the_buckets_record() { + let fake = Arc::new(Fake::default()); + let reads = DirectReads::new(fake.clone(), "", no_keys()); + let record = |format: &str, sse: &str| { + format!( + r#"{{"format":"{format}","encryptionConfiguration":{{"sseAlgorithm":"{sse}"}}}}"# + ) + .into_bytes() + }; + fake.add("/buckets/plain", b"", |e| e.is_directory = true); + fake.add("/buckets/iceberg", b"", |e| { + e.is_directory = true; + e.extended + .insert(TABLE_BUCKET_METADATA.into(), record("ICEBERG", "AES256")); + }); + fake.add("/buckets/lance", b"", |e| { + e.is_directory = true; + e.extended + .insert(TABLE_BUCKET_METADATA.into(), record("LANCE", "AES256")); + }); + fake.add("/buckets/bad", b"", |e| { + e.is_directory = true; + e.extended + .insert(TABLE_BUCKET_METADATA.into(), b"{not json".to_vec()); + }); + assert!(!reads.bucket_encrypts("missing").await.unwrap()); + assert!(!reads.bucket_encrypts("plain").await.unwrap()); + assert!(!reads.bucket_encrypts("iceberg").await.unwrap()); + assert!(reads.bucket_encrypts("lance").await.unwrap()); + reads.bucket_encrypts("bad").await.unwrap_err(); + + fake.add("/buckets/lance", b"", |e| { + e.is_directory = true; + e.extended + .insert(TABLE_BUCKET_METADATA.into(), record("LANCE", "")); + }); + assert!( + !reads.bucket_encrypts("lance").await.unwrap(), + "recreated bucket" + ); + } + + /// A table path is removed only when every component is a plain name. + #[tokio::test] + async fn removing_a_table_refuses_path_tricks() { + let reads = DirectReads::new(Arc::new(Fake::default()), "", no_keys()); + for location in [ + "s3://vb/vector/..", + "s3://vb/./idx", + "s3://../vector/idx", + "s3://vb", + ] { + reads.remove_table(location).await.expect_err(location); + } + } } diff --git a/seaweed-vector/crates/server/src/tls.rs b/seaweed-worker/crates/lance/src/grpc_tls.rs similarity index 99% rename from seaweed-vector/crates/server/src/tls.rs rename to seaweed-worker/crates/lance/src/grpc_tls.rs index e5b8eaf827..99bf1df13c 100644 --- a/seaweed-vector/crates/server/src/tls.rs +++ b/seaweed-worker/crates/lance/src/grpc_tls.rs @@ -220,7 +220,7 @@ fn build_config(files: &GrpcServerTls) -> Result { impl ServerTls { pub fn load(files: GrpcServerTls) -> Result> { - weed_lance_worker::tls::install_default_crypto_provider(); + crate::tls::install_default_crypto_provider(); let loaded = read_all(&files)?; let config = build_config(&files)?; Ok(Arc::new(Self { @@ -400,7 +400,7 @@ fn client_config( files: &GrpcTls, (ca, cert, key): (Vec, Vec, Vec), ) -> Result { - weed_lance_worker::tls::install_default_crypto_provider(); + crate::tls::install_default_crypto_provider(); let verifier = ServerIdentityVerifier { roots: Arc::new(root_store(&files.ca_path, &ca)?), provider: Arc::new(aws_lc_rs::default_provider()), @@ -676,7 +676,7 @@ mod tests { #[tokio::test] async fn a_silent_peer_does_not_hold_up_the_next_handshake() { use futures::StreamExt; - weed_lance_worker::tls::install_default_crypto_provider(); + crate::tls::install_default_crypto_provider(); let dir = tempfile::tempdir().unwrap(); let tls = ServerTls::load(install(dir.path(), "a")).unwrap(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -722,7 +722,7 @@ mod tests { #[test] fn client_common_names_follow_the_allowlist() { - weed_lance_worker::tls::install_default_crypto_provider(); + crate::tls::install_default_crypto_provider(); let mut roots = RootCertStore::empty(); for cert in CertificateDer::pem_file_iter(fixture("ca.pem")).unwrap() { roots.add(cert.unwrap()).unwrap(); @@ -756,7 +756,7 @@ mod tests { #[test] fn server_identities_follow_the_allowlist() { - weed_lance_worker::tls::install_default_crypto_provider(); + crate::tls::install_default_crypto_provider(); let leaf = |name: &str| CertificateDer::from_pem_file(fixture(name)).unwrap(); let verifier = |allowed: &[&str], wildcard: &str| ServerIdentityVerifier { roots: Arc::new( diff --git a/seaweed-worker/crates/lance/src/jobs/cleanup.rs b/seaweed-worker/crates/lance/src/jobs/cleanup.rs index 028739d688..83a992736a 100644 --- a/seaweed-worker/crates/lance/src/jobs/cleanup.rs +++ b/seaweed-worker/crates/lance/src/jobs/cleanup.rs @@ -31,7 +31,7 @@ const MAX_MIN_VERSIONS: i64 = 1000; /// the cluster owned the policy than each writer. pub struct CleanupVersionsHandler { namespace_url: String, - fallback: dataset::FallbackOptions, + storage: dataset::Storage, signing_key: Option, metrics: Option, } @@ -40,7 +40,7 @@ impl CleanupVersionsHandler { pub fn new(namespace_url: String) -> Self { Self { namespace_url, - fallback: dataset::FallbackOptions::new(), + storage: dataset::Storage::default(), signing_key: None, metrics: None, } @@ -51,8 +51,8 @@ impl CleanupVersionsHandler { self } - pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self { - self.fallback = fallback; + pub fn with_storage(mut self, storage: dataset::Storage) -> Self { + self.storage = storage; self } @@ -62,9 +62,7 @@ impl CleanupVersionsHandler { } fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()) - .with_fallback(&self.fallback) - .with_signing_key(self.signing_key.as_ref()) + NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref()) } } @@ -160,7 +158,7 @@ impl JobHandler for CleanupVersionsHandler { let mut proposals = Vec::new(); for encoded in &tables { let id = parse_id(encoded); - let table = match dataset::open(&client, &id, &self.fallback).await { + let table = match dataset::open(&client, &id, &self.storage).await { Ok(table) => table, Err(err) => { if let Some(counters) = &self.metrics { @@ -256,7 +254,7 @@ impl JobHandler for CleanupVersionsHandler { ) as usize; let client = self.client(); - let table = dataset::open(&client, &id, &self.fallback).await?; + let table = dataset::open(&client, &id, &self.storage).await?; let before = table.stats().await?; // The floor is a promise about how much history survives, so it has to diff --git a/seaweed-worker/crates/lance/src/jobs/compact.rs b/seaweed-worker/crates/lance/src/jobs/compact.rs index a4ae5d7565..4d39dcd919 100644 --- a/seaweed-worker/crates/lance/src/jobs/compact.rs +++ b/seaweed-worker/crates/lance/src/jobs/compact.rs @@ -34,7 +34,7 @@ const MIN_FRAGMENTS_CEILING: i64 = 4096; /// accumulates small files the same way an Iceberg table does. pub struct CompactHandler { namespace_url: String, - fallback: dataset::FallbackOptions, + storage: dataset::Storage, signing_key: Option, metrics: Option, } @@ -43,7 +43,7 @@ impl CompactHandler { pub fn new(namespace_url: String) -> Self { Self { namespace_url, - fallback: dataset::FallbackOptions::new(), + storage: dataset::Storage::default(), signing_key: None, metrics: None, } @@ -55,8 +55,8 @@ impl CompactHandler { } /// Storage options to use where the namespace vends none. - pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self { - self.fallback = fallback; + pub fn with_storage(mut self, storage: dataset::Storage) -> Self { + self.storage = storage; self } @@ -66,9 +66,7 @@ impl CompactHandler { } fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()) - .with_fallback(&self.fallback) - .with_signing_key(self.signing_key.as_ref()) + NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref()) } } @@ -154,7 +152,7 @@ impl JobHandler for CompactHandler { let mut observations = Vec::new(); for encoded in &tables { let id = parse_id(encoded); - let table = match dataset::open(&client, &id, &self.fallback).await { + let table = match dataset::open(&client, &id, &self.storage).await { Ok(table) => table, Err(err) => { // A table that cannot be opened is the next run's problem, @@ -271,7 +269,7 @@ impl JobHandler for CompactHandler { let client = self.client(); // Re-resolve rather than trusting the location detection saw: the table // may have been repointed, and the vended credentials have expired. - let mut table = dataset::open(&client, &id, &self.fallback).await?; + let mut table = dataset::open(&client, &id, &self.storage).await?; let before = table.stats().await?; sender.send_progress(JobProgressUpdate { diff --git a/seaweed-worker/crates/lance/src/jobs/indices.rs b/seaweed-worker/crates/lance/src/jobs/indices.rs index 87ce12d723..3d396a4a58 100644 --- a/seaweed-worker/crates/lance/src/jobs/indices.rs +++ b/seaweed-worker/crates/lance/src/jobs/indices.rs @@ -38,7 +38,7 @@ const MAX_UNINDEXED_CEILING: i64 = 100_000_000; /// slow one. pub struct OptimizeIndicesHandler { namespace_url: String, - fallback: dataset::FallbackOptions, + storage: dataset::Storage, signing_key: Option, metrics: Option, } @@ -47,7 +47,7 @@ impl OptimizeIndicesHandler { pub fn new(namespace_url: String) -> Self { Self { namespace_url, - fallback: dataset::FallbackOptions::new(), + storage: dataset::Storage::default(), signing_key: None, metrics: None, } @@ -58,8 +58,8 @@ impl OptimizeIndicesHandler { self } - pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self { - self.fallback = fallback; + pub fn with_storage(mut self, storage: dataset::Storage) -> Self { + self.storage = storage; self } @@ -69,9 +69,7 @@ impl OptimizeIndicesHandler { } fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()) - .with_fallback(&self.fallback) - .with_signing_key(self.signing_key.as_ref()) + NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref()) } } @@ -202,7 +200,7 @@ impl JobHandler for OptimizeIndicesHandler { let mut proposals = Vec::new(); for encoded in &tables { let id = parse_id(encoded); - let table = match dataset::open(&client, &id, &self.fallback).await { + let table = match dataset::open(&client, &id, &self.storage).await { Ok(table) => table, Err(err) => { if let Some(counters) = &self.metrics { @@ -284,7 +282,7 @@ impl JobHandler for OptimizeIndicesHandler { .ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?; let client = self.client(); - let mut table = dataset::open(&client, &id, &self.fallback).await?; + let mut table = dataset::open(&client, &id, &self.storage).await?; let vector_table = is_vector_table(&id); let before = index_rows(&table, vector_table) .await? diff --git a/seaweed-worker/crates/lance/src/jobs/mod.rs b/seaweed-worker/crates/lance/src/jobs/mod.rs index b86c8b0ab5..f654a98bab 100644 --- a/seaweed-worker/crates/lance/src/jobs/mod.rs +++ b/seaweed-worker/crates/lance/src/jobs/mod.rs @@ -42,38 +42,38 @@ pub(crate) fn table_id(parameters: &HashMap) -> Option, metrics: Option, ) -> Vec> { vec![ Arc::new( compact::CompactHandler::new(namespace_url.clone()) - .with_fallback(fallback.clone()) + .with_storage(storage.clone()) .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( indices::OptimizeIndicesHandler::new(namespace_url.clone()) - .with_fallback(fallback.clone()) + .with_storage(storage.clone()) .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( vector_index::VectorIndexHandler::new(namespace_url.clone()) - .with_fallback(fallback.clone()) + .with_storage(storage.clone()) .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( sort::SortHandler::new(namespace_url.clone()) - .with_fallback(fallback.clone()) + .with_storage(storage.clone()) .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( cleanup::CleanupVersionsHandler::new(namespace_url) - .with_fallback(fallback) + .with_storage(storage) .with_signing_key(signing_key) .with_metrics(metrics), ), diff --git a/seaweed-worker/crates/lance/src/jobs/sort.rs b/seaweed-worker/crates/lance/src/jobs/sort.rs index 5ac6859f6a..bc4397dd02 100644 --- a/seaweed-worker/crates/lance/src/jobs/sort.rs +++ b/seaweed-worker/crates/lance/src/jobs/sort.rs @@ -53,7 +53,7 @@ const BYTES_PER_MB: u64 = 1024 * 1024; /// rewrite restores it once rows have been appended. pub struct SortHandler { namespace_url: String, - fallback: dataset::FallbackOptions, + storage: dataset::Storage, signing_key: Option, metrics: Option, } @@ -62,7 +62,7 @@ impl SortHandler { pub fn new(namespace_url: String) -> Self { Self { namespace_url, - fallback: dataset::FallbackOptions::new(), + storage: dataset::Storage::default(), signing_key: None, metrics: None, } @@ -74,8 +74,8 @@ impl SortHandler { } /// Storage options to use where the namespace vends none. - pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self { - self.fallback = fallback; + pub fn with_storage(mut self, storage: dataset::Storage) -> Self { + self.storage = storage; self } @@ -85,9 +85,7 @@ impl SortHandler { } fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()) - .with_fallback(&self.fallback) - .with_signing_key(self.signing_key.as_ref()) + NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref()) } } @@ -207,7 +205,7 @@ impl JobHandler for SortHandler { let mut observations = Vec::new(); for encoded in &tables { let id = parse_id(encoded); - let table = match dataset::open(&client, &id, &self.fallback).await { + let table = match dataset::open(&client, &id, &self.storage).await { Ok(table) => table, Err(err) => { // One unreadable table must not end the sweep: the tables @@ -361,7 +359,7 @@ impl JobHandler for SortHandler { // Re-resolve rather than trusting what detection saw: the table may have // been repointed, its declared order changed, and the vended credentials // have expired. - let table = dataset::open(&client, &id, &self.fallback).await?; + let table = dataset::open(&client, &id, &self.storage).await?; let before = table.stats().await?; let config = table.dataset.config().clone(); let spec = resolve( diff --git a/seaweed-worker/crates/lance/src/jobs/vector_index.rs b/seaweed-worker/crates/lance/src/jobs/vector_index.rs index 9941907e2d..413bb6ff18 100644 --- a/seaweed-worker/crates/lance/src/jobs/vector_index.rs +++ b/seaweed-worker/crates/lance/src/jobs/vector_index.rs @@ -45,7 +45,7 @@ pub(crate) fn declare_build_runtime(parameters: &mut HashMap, metrics: Option, } @@ -54,7 +54,7 @@ impl VectorIndexHandler { pub fn new(namespace_url: String) -> Self { Self { namespace_url, - fallback: dataset::FallbackOptions::new(), + storage: dataset::Storage::default(), signing_key: None, metrics: None, } @@ -65,8 +65,8 @@ impl VectorIndexHandler { self } - pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self { - self.fallback = fallback; + pub fn with_storage(mut self, storage: dataset::Storage) -> Self { + self.storage = storage; self } @@ -76,9 +76,7 @@ impl VectorIndexHandler { } fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()) - .with_fallback(&self.fallback) - .with_signing_key(self.signing_key.as_ref()) + NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref()) } } @@ -179,7 +177,7 @@ impl JobHandler for VectorIndexHandler { if !is_vector_table(&id) { continue; } - let table = match dataset::open(&client, &id, &self.fallback).await { + let table = match dataset::open(&client, &id, &self.storage).await { Ok(table) => table, Err(err) => { if let Some(counters) = &self.metrics { @@ -272,7 +270,7 @@ impl JobHandler for VectorIndexHandler { .ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?; let client = self.client(); - let mut table = dataset::open(&client, &id, &self.fallback).await?; + let mut table = dataset::open(&client, &id, &self.storage).await?; let rows = table.dataset.count_rows(None).await?; let partitions = partitions_for(rows); let mut output: HashMap = HashMap::new(); diff --git a/seaweed-worker/crates/lance/src/lib.rs b/seaweed-worker/crates/lance/src/lib.rs index 113e548fa7..e3d14d3098 100644 --- a/seaweed-worker/crates/lance/src/lib.rs +++ b/seaweed-worker/crates/lance/src/lib.rs @@ -6,13 +6,27 @@ //! three read and rewrite Lance files. This crate is the worker that can. pub mod catalog; +pub mod cluster; pub mod dataset; pub mod direct_read; +pub mod grpc_tls; pub mod jobs; pub mod metrics; pub mod preview; +pub mod security; pub mod sse; pub mod tls; pub mod vector_index; pub use jobs::handlers; + +/// The master's protocol, for the filers it lists. +pub mod master_pb { + tonic::include_proto!("master_pb"); +} +pub mod volume_server_pb { + tonic::include_proto!("volume_server_pb"); +} +pub mod remote_pb { + tonic::include_proto!("remote_pb"); +} diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 0a269217b8..d5613f6e0d 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -36,13 +36,30 @@ struct Args { #[arg(long, default_value = "1")] max_concurrency: i32, - /// Storage credentials to use where the namespace vends none. A gateway - /// with STS configured vends its own and these are ignored. - #[arg(long, env = "WEED_S3_ACCESS_KEY")] - access_key: Option, - - #[arg(long, env = "WEED_S3_SECRET_KEY")] - secret_key: Option, + /// Comma-separated master addresses: the worker asks them for the filers + /// it keeps tables through. + #[arg(long, alias = "master", env = "WEED_MASTERS")] + masters: String, + /// The filer group whose filers the worker uses. + #[arg( + long, + alias = "filer.group", + env = "WEED_FILER_GROUP", + default_value = "" + )] + filer_group: String, + /// Encrypt every chunk written for the volume servers, as `weed s3 + /// -encryptVolumeData` does. + #[arg( + long = "encryptVolumeData", + alias = "encrypt-volume-data", + env = "WEED_ENCRYPT_VOLUME_DATA" + )] + encrypt_volume_data: bool, + /// The cluster's security.toml; by default the file weed processes find + /// in -config_dir, ., ~/.seaweedfs and /etc/seaweedfs. + #[arg(long, env = "WEED_WORKER_SECURITY_TOML")] + security_toml: Option, /// mTLS for the admin stream, the same certificates the Go worker reads /// from the [grpc.worker] section of security.toml. All three together, or @@ -139,12 +156,25 @@ async fn main() -> Result<()> { let args = Args::parse(); let tls = args.tls()?; + // One security.toml for everything: the admin stream's and the catalog's + // signing key, and the cluster connection. + let security = + weed_lance_worker::security::Security::load(args.security_toml.as_deref(), None)?; let filer_signing = match args.filer_signing_key.filter(|key| !key.is_empty()) { Some(key) => Some((key, 0)), - None => seaweed_worker_core::security_config::filer_signing().map(|signing| { - tracing::info!(source = %signing.source, "using jwt.filer_signing.key"); - (signing.key, signing.expires_after_seconds) - }), + None => security + .get("jwt.filer_signing.key") + .filter(|key| !key.is_empty()) + .map(|key| { + if let Some(path) = security.path() { + tracing::info!(source = %path.display(), "using jwt.filer_signing.key"); + } + let expires = security + .get("jwt.filer_signing.expires_after_seconds") + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + (key, expires) + }), }; let (filer_signing_key, configured_expiry) = filer_signing.unzip(); let signing_key = weed_lance_worker::catalog::SigningKey::new(filer_signing_key.clone()); @@ -163,11 +193,11 @@ async fn main() -> Result<()> { ..Default::default() }; - let mut fallback = weed_lance_worker::dataset::FallbackOptions::new(); - if let (Some(access), Some(secret)) = (args.access_key, args.secret_key) { - fallback.insert("aws_access_key_id".to_string(), access); - fallback.insert("aws_secret_access_key".to_string(), secret); - } + let cluster = + weed_lance_worker::cluster::Cluster::new(&args.masters, &args.filer_group, &security)?; + let storage = weed_lance_worker::dataset::Storage(Some( + cluster.storage(&security, args.encrypt_volume_data).await?, + )); let metrics = Metrics::new(&options.worker_id, &options.worker_version)?; let lance_metrics = LanceMetrics::new(&metrics)?; @@ -179,10 +209,10 @@ async fn main() -> Result<()> { } let mut registry = Registry::new().with_preview(Arc::new( - weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()) + weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), storage.clone()) .with_signing_key(signing_key.clone()), )); - for handler in handlers(args.namespace, fallback, signing_key, Some(lance_metrics)) { + for handler in handlers(args.namespace, storage, signing_key, Some(lance_metrics)) { registry = registry.register(handler); } diff --git a/seaweed-worker/crates/lance/src/preview.rs b/seaweed-worker/crates/lance/src/preview.rs index 49dd6e2775..3e21821cee 100644 --- a/seaweed-worker/crates/lance/src/preview.rs +++ b/seaweed-worker/crates/lance/src/preview.rs @@ -11,20 +11,20 @@ use futures::TryStreamExt; use seaweed_worker_core::{Preview, PreviewProvider}; use crate::catalog::{NamespaceClient, SigningKey}; -use crate::dataset::{self, FallbackOptions}; +use crate::dataset::{self, Storage}; use crate::jobs::FORMAT; pub struct LancePreview { namespace_url: String, - fallback: FallbackOptions, + storage: Storage, signing_key: Option, } impl LancePreview { - pub fn new(namespace_url: String, fallback: FallbackOptions) -> Self { + pub fn new(namespace_url: String, storage: Storage) -> Self { Self { namespace_url, - fallback, + storage, signing_key: None, } } @@ -43,9 +43,8 @@ impl PreviewProvider for LancePreview { async fn preview(&self, object_id: &[String], row_limit: usize) -> Result { let client = NamespaceClient::new(self.namespace_url.clone()) - .with_fallback(&self.fallback) .with_signing_key(self.signing_key.as_ref()); - let table = dataset::open(&client, object_id, &self.fallback).await?; + let table = dataset::open(&client, object_id, &self.storage).await?; let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64; let mut scanner = table.dataset.scan(); diff --git a/seaweed-vector/crates/server/src/security.rs b/seaweed-worker/crates/lance/src/security.rs similarity index 99% rename from seaweed-vector/crates/server/src/security.rs rename to seaweed-worker/crates/lance/src/security.rs index 023f2cdbb7..557dd28bd6 100644 --- a/seaweed-vector/crates/server/src/security.rs +++ b/seaweed-worker/crates/lance/src/security.rs @@ -4,9 +4,9 @@ use std::path::{Path, PathBuf}; +use crate::sse::SseConfig; use anyhow::{Context, Result, bail}; use sw_client::HttpsTls; -use weed_lance_worker::sse::SseConfig; const FILE_NAME: &str = "security.toml"; diff --git a/seaweed-worker/crates/lance/src/sse.rs b/seaweed-worker/crates/lance/src/sse.rs index d7c48b0b8a..8920e53813 100644 --- a/seaweed-worker/crates/lance/src/sse.rs +++ b/seaweed-worker/crates/lance/src/sse.rs @@ -1,16 +1,17 @@ //! Decodes what the filer and the S3 gateway encrypt, for reads that bypass -//! the gateway. The filer's volume encryption (`cipher_key`) is AES-256-GCM -//! over the chunk, compressed first when `is_compressed`. The gateway's SSE-S3 -//! is AES-256-CTR per chunk under a per-object key, which the chunk's and the -//! object's metadata carry wrapped (AES-256-GCM) by the cluster's KEK; the KEK -//! comes from `s3.sse.kek`, `s3.sse.key` or the filer's `/etc/s3/sse_kek`, as -//! for the gateway. +//! the gateway, and encodes the same for writes that do. The filer's volume +//! encryption (`cipher_key`) is AES-256-GCM over the chunk, compressed first +//! when `is_compressed`. The gateway's SSE-S3 is AES-256-CTR per chunk under a +//! per-object key, which the chunk's and the object's metadata carry wrapped +//! (AES-256-GCM) by the cluster's KEK; the KEK comes from `s3.sse.kek`, +//! `s3.sse.key` or the filer's `/etc/s3/sse_kek`, as for the gateway. use std::collections::HashMap; use std::io::Read; use std::sync::{Arc, Mutex}; -use aes_gcm::aead::Aead; +use aes_gcm::aead::rand_core::RngCore; +use aes_gcm::aead::{Aead, OsRng}; use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; use anyhow::{Context, Result, bail}; use base64::prelude::{BASE64_STANDARD, Engine}; @@ -103,6 +104,22 @@ impl SseS3Keys { self.kek.is_some() } + /// Wraps an object key under the KEK, as the gateway's key manager does: + /// AES-256-GCM under a fresh nonce. + fn wrap_dek(&self, dek: &[u8; KEY_SIZE]) -> Result<(Vec, [u8; GCM_NONCE_SIZE])> { + let Some(kek) = &self.kek else { + bail!( + "SSE-S3 is not configured: set s3.sse.kek or s3.sse.key for the engine as for the S3 gateway" + ); + }; + let mut nonce = [0u8; GCM_NONCE_SIZE]; + OsRng.fill_bytes(&mut nonce); + let wrapped = Aes256Gcm::new(kek.into()) + .encrypt(&Nonce::from(nonce), dek.as_slice()) + .map_err(|_| anyhow::anyhow!("wrapping the object key"))?; + Ok((wrapped, nonce)) + } + fn unwrap_dek(&self, wrapped: &[u8], nonce: &[u8]) -> Result<[u8; KEY_SIZE]> { let Some(kek) = &self.kek else { bail!( @@ -177,6 +194,22 @@ fn gcm_open(key: &[u8; KEY_SIZE], nonce: &[u8], sealed: &[u8]) -> Result .map_err(|_| anyhow::anyhow!("AES-GCM authentication failed")) } +/// The filer's volume encryption on the way in: `nonce || ciphertext` under a +/// fresh chunk key, which the caller records as the chunk's `cipher_key`. +fn encrypt_chunk(data: &[u8]) -> (Vec, Vec) { + let mut key = [0u8; KEY_SIZE]; + OsRng.fill_bytes(&mut key); + let mut nonce = [0u8; GCM_NONCE_SIZE]; + OsRng.fill_bytes(&mut nonce); + let mut out = nonce.to_vec(); + out.extend( + Aes256Gcm::new((&key).into()) + .encrypt(&Nonce::from(nonce), data) + .expect("AES-GCM encryption of a chunk"), + ); + (out, key.to_vec()) +} + /// The filer's volume encryption: `nonce || ciphertext` under the chunk's key. fn decrypt_chunk(cipher_key: &[u8], data: &[u8]) -> Result> { let key: &[u8; KEY_SIZE] = cipher_key @@ -225,6 +258,31 @@ struct ObjectKey { } impl ObjectKey { + fn generate() -> Self { + let mut key = [0u8; KEY_SIZE]; + OsRng.fill_bytes(&mut key); + let mut iv = [0u8; IV_SIZE]; + OsRng.fill_bytes(&mut iv); + Self { key, iv: Some(iv) } + } + + /// The key as the gateway serializes it, with `iv` as its IV. + fn serialize(&self, keys: &SseS3Keys, key_id: &str, iv: [u8; IV_SIZE]) -> Result> { + let (wrapped, nonce) = keys.wrap_dek(&self.key)?; + let mut mac = as Mac>::new_from_slice(&self.key).expect("any key length"); + mac.update(&iv); + mac.update(AES256.as_bytes()); + let commitment = mac.finalize().into_bytes(); + Ok(serde_json::to_vec(&serde_json::json!({ + "algorithm": AES256, + "keyId": key_id, + "encryptedDEK": BASE64_STANDARD.encode(&wrapped), + "nonce": BASE64_STANDARD.encode(nonce), + "iv": BASE64_STANDARD.encode(iv), + "keyCommitment": BASE64_STANDARD.encode(commitment), + }))?) + } + fn parse(keys: &SseS3Keys, data: &[u8]) -> Result { let meta: SerializedKey = serde_json::from_slice(data).context("SSE-S3 metadata")?; let b64 = |field: &str| { @@ -270,6 +328,98 @@ impl ObjectKey { } } +/// One chunk's bytes as the volume stores them, with what its `FileChunk` +/// records about them. +pub struct EncodedChunk { + pub data: Vec, + pub cipher_key: Vec, + pub sse_type: i32, + pub sse_metadata: Vec, +} + +/// Encodes an object's chunks as the gateway would store them: SSE-S3 when the +/// bucket encrypts, then the filer's volume encryption when the cluster asks +/// for it. The first chunk is under the object's key and IV, which is how the +/// gateway reads an object of one chunk; every other chunk is under a key and +/// IV of its own, as the gateway's multipart parts are and as it reads an +/// object of several, so no two chunks share a keystream whatever their +/// boundaries. +pub struct EntryEncoder { + keys: Arc, + object: Option<(ObjectKey, String)>, + cipher: bool, +} + +fn key_id() -> String { + format!("sse-s3-key-{}", OsRng.next_u64() >> 1) +} + +impl EntryEncoder { + pub fn new(keys: Arc, sse_s3: bool, cipher: bool) -> Result { + let object = if sse_s3 { + if !keys.configured() { + bail!( + "the bucket encrypts with SSE-S3, which needs s3.sse.kek or s3.sse.key for the engine as for the S3 gateway" + ); + } + Some((ObjectKey::generate(), key_id())) + } else { + None + }; + Ok(Self { + keys, + object, + cipher, + }) + } + + /// The bytes of the chunk at `offset` in the object, as they go to the + /// volume server, with the SSE-S3 metadata that reads them back. + pub fn encode(&self, offset: u64, mut data: Vec) -> Result { + let (sse_type, sse_metadata) = match &self.object { + Some((object, key_id)) if offset == 0 => { + let iv = object.iv.expect("a generated key has an IV"); + object.decrypt(0, &mut data)?; + (SSE_S3, object.serialize(&self.keys, key_id, iv)?) + } + Some(_) => { + let chunk_key = ObjectKey::generate(); + let iv = chunk_key.iv.expect("a generated key has an IV"); + chunk_key.decrypt(0, &mut data)?; + (SSE_S3, chunk_key.serialize(&self.keys, &key_id(), iv)?) + } + None => (0, Vec::new()), + }; + let (data, cipher_key) = if self.cipher { + encrypt_chunk(&data) + } else { + (data, Vec::new()) + }; + Ok(EncodedChunk { + data, + cipher_key, + sse_type, + sse_metadata, + }) + } + + /// The attributes the entry carries for its chunks to be decodable: the + /// SSE-S3 header and the object key. + pub fn attributes(&self) -> Result)>> { + let Some((object, key_id)) = &self.object else { + return Ok(Vec::new()); + }; + let iv = object.iv.expect("a generated key has an IV"); + Ok(vec![ + (SSE_HEADER_ATTRIBUTE.to_string(), AES256.as_bytes().to_vec()), + ( + SSE_S3_KEY_ATTRIBUTE.to_string(), + object.serialize(&self.keys, key_id, iv)?, + ), + ]) + } +} + fn attribute<'a>(entry: &'a Entry, name: &str) -> Option<&'a [u8]> { entry .extended diff --git a/seaweed-worker/crates/lance/tests/common/mod.rs b/seaweed-worker/crates/lance/tests/common/mod.rs index 10ffb3db86..510c5597f7 100644 --- a/seaweed-worker/crates/lance/tests/common/mod.rs +++ b/seaweed-worker/crates/lance/tests/common/mod.rs @@ -14,6 +14,7 @@ use seaweed_worker_core::pb::{ JobProgressUpdate, JobProposal, ObjectObservation, WorkerObservations, config_value::Kind, }; use seaweed_worker_core::{DetectionSender, ExecutionSender}; +use weed_lance_worker::catalog::{NamespaceClient, SigningKey}; /// Keeps everything a handler reports, so a test can assert on it afterwards. #[derive(Default)] @@ -57,6 +58,14 @@ impl ExecutionSender for Recorder { /// these handlers rewrite real files, and there is nothing to learn from them /// against a fake. pub fn namespace_url() -> Option { + // The handlers say why they skip a table only in their logs. + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_test_writer() + .try_init(); std::env::var("WEED_LANCE_NAMESPACE") .ok() .filter(|s| !s.is_empty()) @@ -73,25 +82,34 @@ pub fn int_config(name: &str, value: i64) -> HashMap { values } -/// What to reach the data with where the namespace vends nothing of its own - -/// a gateway without STS vends no credentials, and one bound to a wildcard -/// address vends no endpoint either. Taken from the environment so a harness -/// can point these tests at a gateway that does check what it is given. -pub fn fallback() -> weed_lance_worker::dataset::FallbackOptions { - let mut options = weed_lance_worker::dataset::FallbackOptions::new(); - let from_env = - |name: &str, default: &str| std::env::var(name).unwrap_or_else(|_| default.to_string()); - options.insert( - "aws_access_key_id".to_string(), - from_env("AWS_ACCESS_KEY_ID", "any"), - ); - options.insert( - "aws_secret_access_key".to_string(), - from_env("AWS_SECRET_ACCESS_KEY", "any"), - ); - if let Ok(endpoint) = std::env::var("AWS_ENDPOINT_URL") { - options.insert("aws_endpoint".to_string(), endpoint); - options.insert("allow_http".to_string(), "true".to_string()); - } - options +/// The cluster's storage, from the masters WEED_MASTERS names. A live +/// namespace without them would have the tests seed tables through lance's +/// own S3 store, which reaches nothing, so the pair is required together. +pub async fn storage() -> weed_lance_worker::dataset::Storage { + let masters = std::env::var("WEED_MASTERS") + .ok() + .filter(|s| !s.is_empty()) + .expect("WEED_LANCE_NAMESPACE names a live namespace; WEED_MASTERS must name its masters"); + let security = security(); + let cluster = + weed_lance_worker::cluster::Cluster::new(&masters, "", &security).expect("masters"); + weed_lance_worker::dataset::Storage(Some( + cluster.storage(&security, false).await.expect("storage"), + )) +} + +fn security() -> weed_lance_worker::security::Security { + weed_lance_worker::security::Security::load(None, None).expect("security.toml") +} + +/// The cluster token the namespace authenticates its processes by, from +/// `jwt.filer_signing.key` (WEED_JWT_FILER_SIGNING_KEY or security.toml), +/// as the worker reads it. +pub fn signing_key() -> Option { + SigningKey::new(security().get("jwt.filer_signing.key")) +} + +/// The namespace client the handlers would use. +pub fn client(url: &str) -> NamespaceClient { + NamespaceClient::new(url.to_string()).with_signing_key(signing_key().as_ref()) } diff --git a/seaweed-worker/crates/lance/tests/compaction.rs b/seaweed-worker/crates/lance/tests/compaction.rs index 307e3d437a..9851bd2101 100644 --- a/seaweed-worker/crates/lance/tests/compaction.rs +++ b/seaweed-worker/crates/lance/tests/compaction.rs @@ -4,21 +4,18 @@ //! tests skip without Docker: compaction rewrites real files, and there is //! nothing to learn from it against a fake. -use std::collections::HashMap; - use anyhow::Result; use seaweed_worker_core::pb::{ ExecuteJobRequest, JobSpec, RunDetectionRequest, config_value::Kind, }; use seaweed_worker_core::{JobHandler, PreviewProvider}; -use weed_lance_worker::catalog::NamespaceClient; use weed_lance_worker::jobs::cleanup::CleanupVersionsHandler; use weed_lance_worker::jobs::compact::{CompactHandler, JOB_TYPE}; use weed_lance_worker::jobs::indices::OptimizeIndicesHandler; use weed_lance_worker::preview::LancePreview; mod common; -use common::{Recorder, fallback, int_config, namespace_url}; +use common::{Recorder, client, int_config, namespace_url, signing_key, storage}; /// These tests drive one live gateway and one shared catalog: `list_all_tables` /// sweeps everything, so a table another test is writing shows up in this test's @@ -47,7 +44,6 @@ async fn seed_table( }; use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{Dataset, WriteMode, WriteParams}; - use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use std::sync::Arc; // Declaring is the namespace's job, not the worker's, so the test asks for @@ -69,12 +65,11 @@ async fn seed_table( .await? .error_for_status()?; - let client = NamespaceClient::new(url.to_string()); + let client = client(url); let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()]; let description = client.describe_table(&id).await?; - let mut options = description.storage_options.clone(); - options.extend(fallback()); + let wrapper = storage().await.wrapper(&description.location).await?; const DIM: i32 = 16; let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int64, false), @@ -108,12 +103,7 @@ async fn seed_table( } else { WriteMode::Append }, - store_params: Some(ObjectStoreParams { - storage_options_accessor: Some(std::sync::Arc::new( - StorageOptionsAccessor::with_static_options(options.clone()), - )), - ..Default::default() - }), + store_params: Some(weed_lance_worker::dataset::store_params(wrapper.clone())), ..Default::default() }; let dataset = Dataset::write(reader, description.location.as_str(), Some(params)).await?; @@ -154,7 +144,9 @@ async fn compacts_a_fragmented_table() { let encoded = seed_fragmented_table(&url, "compactme", 12) .await .expect("seed a fragmented table"); - let handler = CompactHandler::new(url).with_fallback(fallback()); + let handler = CompactHandler::new(url) + .with_storage(storage().await) + .with_signing_key(signing_key()); let recorder = Recorder::default(); let request = RunDetectionRequest { @@ -242,7 +234,9 @@ async fn cleans_up_old_versions() { .await .expect("seed a table with versions to clean"); - let handler = CleanupVersionsHandler::new(url).with_fallback(fallback()); + let handler = CleanupVersionsHandler::new(url) + .with_storage(storage().await) + .with_signing_key(signing_key()); let recorder = Recorder::default(); let request = RunDetectionRequest { @@ -306,7 +300,9 @@ async fn skips_tables_without_indices() { .await .expect("seed a table without an index"); - let handler = OptimizeIndicesHandler::new(url).with_fallback(fallback()); + let handler = OptimizeIndicesHandler::new(url) + .with_storage(storage().await) + .with_signing_key(signing_key()); let recorder = Recorder::default(); let request = RunDetectionRequest { @@ -345,7 +341,9 @@ async fn reindexes_rows_an_index_does_not_cover() { let encoded = seed_table(&url, "reindexme", 2, 512, true) .await .expect("seed an indexed table with uncovered rows"); - let handler = OptimizeIndicesHandler::new(url).with_fallback(fallback()); + let handler = OptimizeIndicesHandler::new(url) + .with_storage(storage().await) + .with_signing_key(signing_key()); let recorder = Recorder::default(); let request = RunDetectionRequest { @@ -418,7 +416,7 @@ async fn previews_rows_of_a_table() { .await .expect("seed a table to preview"); - let provider = LancePreview::new(url, fallback()); + let provider = LancePreview::new(url, storage().await); let id = vec!["vec".to_string(), "ml".to_string(), "previewme".to_string()]; let preview = provider.preview(&id, 4).await.expect("preview the table"); @@ -450,27 +448,32 @@ async fn concurrent_writers_keep_every_commit() { .await .expect("seed the table the writers will append to"); - let client = NamespaceClient::new(url.clone()); + let client = client(&url); let id = vec!["vec".to_string(), "ml".to_string(), "racers".to_string()]; let description = client.describe_table(&id).await.expect("describe"); - let mut options = description.storage_options.clone(); - options.extend(fallback()); + let wrapper = storage() + .await + .wrapper(&description.location) + .await + .expect("storage for the table"); - let writes = (0..WRITERS).map(|writer| { - let location = description.location.clone(); - let options = options.clone(); - tokio::spawn( - async move { append_rows(&location, &options, writer * 1000, ROWS_EACH).await }, - ) - }); - for (writer, handle) in writes.enumerate() { + let writes: Vec<_> = (0..WRITERS) + .map(|writer| { + let location = description.location.clone(); + let wrapper = wrapper.clone(); + tokio::spawn( + async move { append_rows(&location, wrapper, writer * 1000, ROWS_EACH).await }, + ) + }) + .collect(); + for (writer, handle) in writes.into_iter().enumerate() { handle .await .expect("writer panicked") .unwrap_or_else(|err| panic!("writer {writer} failed to commit: {err:#}")); } - let table = weed_lance_worker::dataset::open(&client, &id, &fallback()) + let table = weed_lance_worker::dataset::open(&client, &id, &storage().await) .await .expect("reopen the table"); let rows = table.dataset.count_rows(None).await.expect("count rows"); @@ -484,7 +487,7 @@ async fn concurrent_writers_keep_every_commit() { /// Appends one batch to an existing dataset, the way an independent writer would. async fn append_rows( location: &str, - options: &HashMap, + wrapper: Option>, first_id: i64, rows: i64, ) -> Result<()> { @@ -493,7 +496,6 @@ async fn append_rows( }; use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{Dataset, WriteMode, WriteParams}; - use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use std::sync::Arc; const DIM: i32 = 16; @@ -522,12 +524,7 @@ async fn append_rows( )?; let params = WriteParams { mode: WriteMode::Append, - store_params: Some(ObjectStoreParams { - storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - options.clone(), - ))), - ..Default::default() - }), + store_params: Some(weed_lance_worker::dataset::store_params(wrapper)), ..Default::default() }; Dataset::write( diff --git a/seaweed-worker/crates/lance/tests/lifecycle.rs b/seaweed-worker/crates/lance/tests/lifecycle.rs index 9b5253fff2..8a32884207 100644 --- a/seaweed-worker/crates/lance/tests/lifecycle.rs +++ b/seaweed-worker/crates/lance/tests/lifecycle.rs @@ -18,7 +18,7 @@ use weed_lance_worker::jobs::cleanup::{self, CleanupVersionsHandler}; use weed_lance_worker::jobs::compact::{CompactHandler, JOB_TYPE as COMPACT_JOB_TYPE}; mod common; -use common::{Recorder, fallback, int_config, namespace_url}; +use common::{Recorder, int_config, namespace_url, signing_key, storage}; fn table() -> Option { std::env::var("WEED_LANCE_TABLE") @@ -40,14 +40,18 @@ async fn maintains_the_named_table() { // anything above one fragment is worth merging here. let mut config = int_config("min_fragments", 2); config.extend(int_config("target_rows_per_fragment", 1_048_576)); - let compact = CompactHandler::new(url.clone()).with_fallback(fallback()); + let compact = CompactHandler::new(url.clone()) + .with_storage(storage().await) + .with_signing_key(signing_key()); run(&compact, COMPACT_JOB_TYPE, &name, config, true).await; // Keep the current version and retain nothing else, so the versions // compaction superseded are actually removed rather than counted. let mut config = int_config("min_versions_to_keep", 1); config.extend(int_config("retain_hours", 0)); - let cleanup = CleanupVersionsHandler::new(url).with_fallback(fallback()); + let cleanup = CleanupVersionsHandler::new(url) + .with_storage(storage().await) + .with_signing_key(signing_key()); run(&cleanup, cleanup::JOB_TYPE, &name, config, true).await; } diff --git a/seaweed-worker/crates/lance/tests/sort.rs b/seaweed-worker/crates/lance/tests/sort.rs index 9b76d5febf..06e1d7d707 100644 --- a/seaweed-worker/crates/lance/tests/sort.rs +++ b/seaweed-worker/crates/lance/tests/sort.rs @@ -9,11 +9,10 @@ use seaweed_worker_core::JobHandler; use seaweed_worker_core::pb::{ ConfigValue, ExecuteJobRequest, JobSpec, RunDetectionRequest, config_value::Kind, }; -use weed_lance_worker::catalog::NamespaceClient; use weed_lance_worker::jobs::sort::{JOB_TYPE, SortHandler}; mod common; -use common::{Recorder, fallback, namespace_url}; +use common::{Recorder, client, namespace_url, signing_key, storage}; /// One live gateway and one shared catalog, and `list_all_tables` sweeps /// everything, so these tests take a lock the way the compaction ones do. @@ -52,7 +51,6 @@ async fn seed_unsorted_table( use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator}; use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{Dataset, WriteMode, WriteParams}; - use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use std::sync::Arc; let http = reqwest::Client::new(); @@ -70,11 +68,10 @@ async fn seed_unsorted_table( .await? .error_for_status()?; - let client = NamespaceClient::new(url.to_string()); + let client = client(url); let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()]; let description = client.describe_table(&id).await?; - let mut options = description.storage_options.clone(); - options.extend(fallback()); + let wrapper = storage().await.wrapper(&description.location).await?; let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); let total = batches * rows_each; @@ -92,12 +89,7 @@ async fn seed_unsorted_table( } else { WriteMode::Append }, - store_params: Some(ObjectStoreParams { - storage_options_accessor: Some(Arc::new( - StorageOptionsAccessor::with_static_options(options.clone()), - )), - ..Default::default() - }), + store_params: Some(weed_lance_worker::dataset::store_params(wrapper.clone())), ..Default::default() }; dataset = Some( @@ -125,9 +117,9 @@ async fn stored_ids(url: &str, name: &str) -> Result> { use arrow_array::Int64Array; use futures::TryStreamExt; - let client = NamespaceClient::new(url.to_string()); + let client = client(url); let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()]; - let table = weed_lance_worker::dataset::open(&client, &id, &fallback()).await?; + let table = weed_lance_worker::dataset::open(&client, &id, &storage().await).await?; let stream = table.dataset.scan().try_into_stream().await?; let batches: Vec<_> = stream.try_collect().await?; let mut ids = Vec::new(); @@ -164,7 +156,9 @@ async fn sorts_a_table_and_stops_proposing_it() { "the seeded table was already sorted, so the test proves nothing" ); - let handler = SortHandler::new(url.clone()).with_fallback(fallback()); + let handler = SortHandler::new(url.clone()) + .with_storage(storage().await) + .with_signing_key(signing_key()); let recorder = Recorder::default(); let detection = RunDetectionRequest { request_id: "detect-sort".to_string(), @@ -259,7 +253,9 @@ async fn leaves_a_table_that_declares_no_order_alone() { .await .expect("seed a table with no declared order"); - let handler = SortHandler::new(url.clone()).with_fallback(fallback()); + let handler = SortHandler::new(url.clone()) + .with_storage(storage().await) + .with_signing_key(signing_key()); let recorder = Recorder::default(); handler .detect( @@ -302,7 +298,9 @@ async fn proposes_a_replacement_that_barely_changes_the_row_count() { .await .expect("seed an unsorted table"); - let handler = SortHandler::new(url.clone()).with_fallback(fallback()); + let handler = SortHandler::new(url.clone()) + .with_storage(storage().await) + .with_signing_key(signing_key()); // A threshold no realistic append would cross, so nothing but the identity // of the data can be what proposes this table. let detection = RunDetectionRequest { @@ -395,7 +393,9 @@ async fn deletions_do_not_hide_appended_rows() { .await .expect("seed an unsorted table"); - let handler = SortHandler::new(url.clone()).with_fallback(fallback()); + let handler = SortHandler::new(url.clone()) + .with_storage(storage().await) + .with_signing_key(signing_key()); let detection = RunDetectionRequest { request_id: "detect-delete".to_string(), job_type: JOB_TYPE.to_string(), @@ -436,9 +436,9 @@ async fn deletions_do_not_hide_appended_rows() { // Delete 900 of the 1000 sorted rows, then append 200 unsorted ones. The // table is smaller than when it was sorted, so net growth reads as zero. - let client = NamespaceClient::new(url.clone()); + let client = client(&url); let id = vec!["vec".to_string(), "ml".to_string(), "deleteme".to_string()]; - let mut table = weed_lance_worker::dataset::open(&client, &id, &fallback()) + let mut table = weed_lance_worker::dataset::open(&client, &id, &storage().await) .await .expect("reopen the sorted table"); table @@ -472,26 +472,19 @@ async fn append_ids(url: &str, name: &str, first_id: i64, rows: i64) -> Result<( use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator}; use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{Dataset, WriteMode, WriteParams}; - use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use std::sync::Arc; - let client = NamespaceClient::new(url.to_string()); + let client = client(url); let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()]; let description = client.describe_table(&id).await?; - let mut options = description.storage_options.clone(); - options.extend(fallback()); + let wrapper = storage().await.wrapper(&description.location).await?; let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); let ids: Vec = (0..rows).map(|row| first_id + row).collect(); let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(ids))])?; let params = WriteParams { mode: WriteMode::Append, - store_params: Some(ObjectStoreParams { - storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - options, - ))), - ..Default::default() - }), + store_params: Some(weed_lance_worker::dataset::store_params(wrapper)), ..Default::default() }; Dataset::write( diff --git a/seaweed-vector/crates/server/tests/tls/a.key b/seaweed-worker/crates/lance/tests/tls/a.key similarity index 100% rename from seaweed-vector/crates/server/tests/tls/a.key rename to seaweed-worker/crates/lance/tests/tls/a.key diff --git a/seaweed-vector/crates/server/tests/tls/a.pem b/seaweed-worker/crates/lance/tests/tls/a.pem similarity index 100% rename from seaweed-vector/crates/server/tests/tls/a.pem rename to seaweed-worker/crates/lance/tests/tls/a.pem diff --git a/seaweed-vector/crates/server/tests/tls/b.key b/seaweed-worker/crates/lance/tests/tls/b.key similarity index 100% rename from seaweed-vector/crates/server/tests/tls/b.key rename to seaweed-worker/crates/lance/tests/tls/b.key diff --git a/seaweed-vector/crates/server/tests/tls/b.pem b/seaweed-worker/crates/lance/tests/tls/b.pem similarity index 100% rename from seaweed-vector/crates/server/tests/tls/b.pem rename to seaweed-worker/crates/lance/tests/tls/b.pem diff --git a/seaweed-vector/crates/server/tests/tls/ca.pem b/seaweed-worker/crates/lance/tests/tls/ca.pem similarity index 100% rename from seaweed-vector/crates/server/tests/tls/ca.pem rename to seaweed-worker/crates/lance/tests/tls/ca.pem diff --git a/seaweed-worker/crates/sw-client/src/filer.rs b/seaweed-worker/crates/sw-client/src/filer.rs index 7cad70d1e7..d2244cc2da 100644 --- a/seaweed-worker/crates/sw-client/src/filer.rs +++ b/seaweed-worker/crates/sw-client/src/filer.rs @@ -824,7 +824,11 @@ impl FilerClient { .context("CreateEntry RPC failed")? .into_inner(); if !resp.error.is_empty() { - bail!("CreateEntry: {}", resp.error); + let error = anyhow::anyhow!("CreateEntry: {}", resp.error); + if resp.error_code == crate::filer_pb::FilerError::EntryAlreadyExists as i32 { + return Err(error.context(EntryExists)); + } + return Err(error); } Ok(LogPos { ts_ns: resp.log_ts_ns, @@ -1055,6 +1059,24 @@ fn assign_volume_request( /// Entries fetched per `ListEntries` page. const LIST_PAGE_SIZE: u32 = 1000; +/// Marker [`FilerClient::create_entry`] attaches when `o_excl` found the entry +/// already there; detect it with [`is_entry_exists`]. +#[derive(Debug)] +pub struct EntryExists; + +impl std::fmt::Display for EntryExists { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "the entry already exists") + } +} + +impl std::error::Error for EntryExists {} + +/// Whether `err` is an exclusive create that found its entry in place. +pub fn is_entry_exists(err: &anyhow::Error) -> bool { + err.downcast_ref::().is_some() +} + /// Whether `err` is a filer being unreachable or slow rather than refusing /// the request, so that another filer may answer it. pub fn is_unavailable(err: &anyhow::Error) -> bool { diff --git a/seaweed-worker/crates/sw-client/src/lib.rs b/seaweed-worker/crates/sw-client/src/lib.rs index 3eceed004f..28b8361934 100644 --- a/seaweed-worker/crates/sw-client/src/lib.rs +++ b/seaweed-worker/crates/sw-client/src/lib.rs @@ -29,8 +29,8 @@ pub mod write; pub const MAX_READ_BYTES: u64 = 1 << 30; /* 1 GiB */ pub use filer::{ - AssignVolumeOptions, Assignment, EntryPage, FilerClient, FilerConfig, GrpcTls, LogPos, Seek, - VolumeLocation, is_unavailable, seek_page, + AssignVolumeOptions, Assignment, EntryExists, EntryPage, FilerClient, FilerConfig, GrpcTls, + LogPos, Seek, VolumeLocation, is_entry_exists, is_unavailable, seek_page, }; pub use location_cache::{DEFAULT_RETAIN_GENERATIONS, LocationCache}; pub use read::{ diff --git a/seaweed-worker/crates/sw-client/src/transport.rs b/seaweed-worker/crates/sw-client/src/transport.rs index d3f85e4b7b..e111b0ea38 100644 --- a/seaweed-worker/crates/sw-client/src/transport.rs +++ b/seaweed-worker/crates/sw-client/src/transport.rs @@ -105,6 +105,13 @@ pub trait VolumeTransport: Send + Sync { .await } + /// Delete needle `file_id` at `location`; `auth` is the write JWT its + /// assignment came with (`""` if none). A needle already gone is no error. + async fn delete(&self, location: &str, file_id: &str, auth: &str) -> Result<()> { + let _ = (location, file_id, auth); + bail!("{} transport cannot delete needles", self.name()) + } + /// Short label for logs/diagnostics (e.g. `"http"`, `"rdma"`). fn name(&self) -> &'static str; } @@ -342,6 +349,19 @@ impl VolumeTransport for HttpTransport { Ok(VolumeWriteReceipt::default()) } + async fn delete(&self, location: &str, file_id: &str, auth: &str) -> Result<()> { + let url = format!("{}://{location}/{file_id}", self.scheme); + let mut req = self.http.delete(&url).timeout(self.write_timeout); + if !auth.is_empty() { + req = req.header(AUTHORIZATION, format!("BEARER {auth}")); + } + let resp = req.send().await.with_context(|| format!("delete {url}"))?; + if resp.status().is_success() || resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + bail!("delete {url} -> HTTP {}", resp.status()) + } + fn name(&self) -> &'static str { "http" } diff --git a/test/s3tables/lifecycle/cluster_test.go b/test/s3tables/lifecycle/cluster_test.go index ce5037f6b5..98e15b11f3 100644 --- a/test/s3tables/lifecycle/cluster_test.go +++ b/test/s3tables/lifecycle/cluster_test.go @@ -101,6 +101,10 @@ type environment struct { lancePort int adminPort int + // The cluster's jwt.filer_signing.key, which the Lance namespace + // authenticates the Rust worker by. + clusterKey string + weedCancel context.CancelFunc weedCmd *exec.Cmd } @@ -159,6 +163,7 @@ func newEnvironment() (*environment, error) { icebergPort: ports[8], lancePort: ports[9], adminPort: adminPorts[0], + clusterKey: randomSuffix() + randomSuffix(), }, nil } @@ -197,6 +202,7 @@ func (env *environment) start() error { cmd.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID="+accessKey, "AWS_SECRET_ACCESS_KEY="+secretKey, + "WEED_JWT_FILER_SIGNING_KEY="+env.clusterKey, ) if err := cmd.Start(); err != nil { diff --git a/test/s3tables/lifecycle/lance_lifecycle_test.go b/test/s3tables/lifecycle/lance_lifecycle_test.go index 31b5b6c173..8b5d05b008 100644 --- a/test/s3tables/lifecycle/lance_lifecycle_test.go +++ b/test/s3tables/lifecycle/lance_lifecycle_test.go @@ -83,13 +83,14 @@ func (env *environment) maintainLanceTable(t *testing.T, bucket, namespace, tabl cmd := exec.CommandContext(ctx, "cargo", "test", "-p", "weed-lance-worker", "--test", "lifecycle", "--", "--nocapture") cmd.Dir = filepath.Join(env.rootDir, "seaweed-worker") + // The worker keeps the table through the filers and volume servers the + // masters name, and the namespace knows it by the cluster key; it holds + // no S3 credentials. cmd.Env = append(cmd.Environ(), fmt.Sprintf("WEED_LANCE_NAMESPACE=http://127.0.0.1:%d", env.lancePort), fmt.Sprintf("WEED_LANCE_TABLE=%s$%s$%s", bucket, namespace, table), - "AWS_ACCESS_KEY_ID="+accessKey, - "AWS_SECRET_ACCESS_KEY="+secretKey, - "AWS_REGION=us-east-1", - fmt.Sprintf("AWS_ENDPOINT_URL=http://127.0.0.1:%d", env.s3Port), + fmt.Sprintf("WEED_MASTERS=127.0.0.1:%d.%d", env.masterPort, env.masterGrpcPort), + "WEED_JWT_FILER_SIGNING_KEY="+env.clusterKey, ) out, err := cmd.CombinedOutput() t.Logf("lance worker:\n%s", out) diff --git a/test/s3vectors/DESIGN.md b/test/s3vectors/DESIGN.md index 954eaaf2e6..92ffabb11b 100644 --- a/test/s3vectors/DESIGN.md +++ b/test/s3vectors/DESIGN.md @@ -45,10 +45,10 @@ engine RPC, so they work against a plain `weed server`: ### Tier 2 — vector engine required (skipped when absent) `TestVectorDataPlaneLifecycle` spawns a real `weed-vector` against the -environment's S3 endpoint, mirroring the cluster the pytest suite builds in -`conftest.py`: `--masters` points at `master.http.grpc`, and the storage -options carry the same test credentials the IAM config writes, with -`aws_endpoint` at the S3 port. The binary resolves as +environment's cluster, mirroring the cluster the pytest suite builds in +`conftest.py`: `--masters` points at `master.http.grpc`, which is all the +engine needs, since it keeps tables on the filers and volume servers the +masters list and holds no S3 credentials. The binary resolves as `VECTOR_BINARY` env → `./weed-vector` (the Makefile's output) → PATH; the test skips when none is found, and `waitForEngineDiscovery` polls a data-plane op until the gateway routes to the registered engine instead of diff --git a/test/s3vectors/conftest.py b/test/s3vectors/conftest.py index 1fd95c49fc..bdea56ce31 100644 --- a/test/s3vectors/conftest.py +++ b/test/s3vectors/conftest.py @@ -98,7 +98,7 @@ def cluster(request, tmp_path_factory): @pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p)) def open_cluster(request, tmp_path_factory): - """A cluster with no S3 identities, and an engine given no credentials.""" + """A cluster whose gateway has no S3 identities.""" yield from run_cluster(request.param, tmp_path_factory, identities=False) @@ -260,23 +260,29 @@ def run_cluster(param, tmp_path_factory, identities): # VECTOR_READ_JWT=1 makes volume reads need a JWT; every weed process # and the workers sign with the key. read_jwt_key = os.environ.get("VECTOR_READ_JWT") and uuid.uuid4().hex + # The cluster's filer signing key: the gateway's catalog authenticates + # the worker and the CLI by the token they derive from it. + filer_signing_key = uuid.uuid4().hex # VECTOR_TLS=1 puts the volume servers behind HTTPS with client # certificates; every process finds security.toml in its cwd. if os.environ.get("VECTOR_TLS"): write_tls(directory) - def start(name, args, unset=()): - log = stack.enter_context((directory / f"{name}.log").open("w")) + def process_env(unset=()): env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test", AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true", - WEED_S3_SSE_KEY="s3vectors-test", + WEED_S3_SSE_KEY="s3vectors-test", WEED_JWT_FILER_SIGNING_KEY=filer_signing_key, **({"WEED_JWT_SIGNING_READ_KEY": read_jwt_key} if read_jwt_key else {})) env.pop("WEED_S3_SSE_KEK", None) if not read_jwt_key: env.pop("WEED_JWT_SIGNING_READ_KEY", None) for key in unset: env.pop(key) - process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env) + return env + + def start(name, args, unset=()): + log = stack.enter_context((directory / f"{name}.log").open("w")) + process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=process_env(unset)) processes[name] = process stack.callback(stop, process) return process @@ -300,16 +306,11 @@ def run_cluster(param, tmp_path_factory, identities): f"-defaultStoreDir={directory / 'filer'}"]) wait_for(lambda: listening(filer_grpc), processes, "filer not ready") endpoint = f"http://127.0.0.1:{s3}" - # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity, and the engine - # would find it as a credential. + # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity. if identities: config, unset = [f"-config={HERE / 's3.json'}"], () - # The engine uses its credentials only with the endpoint it is given. - credentials = ["--storage-option", "aws_access_key_id=test", - "--storage-option", "aws_secret_access_key=test", - "--storage-option", f"aws_endpoint={endpoint}"] else: - config, unset, credentials = [], ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"), [] + config, unset = [], ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY") # VECTOR_ENCRYPT_VOLUMES=1 has the gateway encrypt every chunk it # stores, which direct reads must undo. encrypt = ["-encryptVolumeData"] if os.environ.get("VECTOR_ENCRYPT_VOLUMES") else [] @@ -320,7 +321,10 @@ def run_cluster(param, tmp_path_factory, identities): wait_for(lambda: listening(s3), processes, "s3 not ready") if os.environ.get("VECTOR_TLS") or read_jwt_key or encrypt: probe_hardening(endpoint, filer, vol, directory, read_jwt_key) - start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", *credentials], unset) + # The engine keeps tables on the filers and volume servers itself; it + # has no S3 credentials to be given, and encrypts its chunks for the + # volume servers as the gateway does. + start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", *encrypt]) wait_for(lambda: listening(vector), processes, "vector engine not ready") if os.environ.get("VECTOR_TLS"): wait_for(lambda: listening(s3_grpc), processes, "s3 gRPC not ready") @@ -358,13 +362,14 @@ def run_cluster(param, tmp_path_factory, identities): wait_for(lambda: http_ready(f"{admin_url}/api/plugin/job-types"), processes, "admin not ready") start("lance-worker", [lance_worker, "--admin", f"127.0.0.1:{admin}.{admin_grpc}", "--namespace", f"http://127.0.0.1:{lance}", "--id", "vectors-test-worker", - "--access-key", "test", "--secret-key", "test"]) + "--masters", master_addr, *(["--encryptVolumeData"] if encrypt else [])]) wait_for(lambda: "lance_vector_index" in [jt["job_type"] for jt in json.load(urllib.request.urlopen(f"{admin_url}/api/plugin/job-types", timeout=2))], processes, "the Lance worker never registered with admin") - yield {"endpoint": endpoint, "mode": mode, "processes": processes, "directory": directory, - "catalog": f"http://127.0.0.1:{lance}", "admin": admin_url} + yield {"endpoint": endpoint, "mode": mode, "processes": processes, "directory": directory, "filer": filer, + "masters": master_addr, "filer_signing_key": filer_signing_key, + "catalog": f"http://127.0.0.1:{lance}", "admin": admin_url, "env": process_env()} @pytest.fixture(scope="module") diff --git a/test/s3vectors/perf_test.py b/test/s3vectors/perf_test.py index 5b7652c4db..b1b039f0b3 100644 --- a/test/s3vectors/perf_test.py +++ b/test/s3vectors/perf_test.py @@ -193,11 +193,13 @@ def test_build_index(perf): pytest.skip("PERF_INDEX not set") cli = Path(os.environ.get("VECTOR_CLI_BINARY", HERE / "weed-vector-cli")).resolve() assert cli.is_file(), f"missing {cli}; run make ./weed-vector-cli" - command = [str(cli), "--namespace", perf["cluster"]["catalog"], "--access-key", "test", - "--secret-key", "test", "index", "build", f"{perf['bucket']}$vector${INDEX}"] + command = [str(cli), "--namespace", perf["cluster"]["catalog"], "--masters", perf["cluster"]["masters"], + "--filer-signing-key", perf["cluster"]["filer_signing_key"], + "index", "build", f"{perf['bucket']}$vector${INDEX}"] if os.environ.get("PERF_PARTITIONS"): command += ["--partitions", os.environ["PERF_PARTITIONS"]] - seconds, out = timed(lambda: subprocess.run(command, check=True, capture_output=True, text=True)) + seconds, out = timed(lambda: subprocess.run(command, check=True, capture_output=True, text=True, + cwd=perf["cluster"]["directory"], env=perf["cluster"]["env"])) report = json.loads(out.stdout) coverage = report["coverage"] assert len(coverage) == 1 and coverage[0]["unindexed_rows"] == 0, coverage diff --git a/test/s3vectors/s3vectors_integration_test.go b/test/s3vectors/s3vectors_integration_test.go index 93494577df..2d2c16cbe6 100644 --- a/test/s3vectors/s3vectors_integration_test.go +++ b/test/s3vectors/s3vectors_integration_test.go @@ -182,9 +182,6 @@ func (e *TestEnvironment) startVectorEngine(t *testing.T, bin string) { cmd := exec.Command(bin, "--masters", masters, "--listen", fmt.Sprintf("%s:%d", e.bindIP, e.vectorPort), - "--storage-option", "aws_access_key_id="+e.accessKey, - "--storage-option", "aws_secret_access_key="+e.secretKey, - "--storage-option", fmt.Sprintf("aws_endpoint=http://%s:%d", e.bindIP, e.s3Port), ) cmd.Dir = e.dataDir log, err := os.Create(filepath.Join(e.dataDir, "weed-vector.log")) diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index 9129e2b3a0..cf0899ff76 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -116,6 +116,42 @@ def index(vectors): vectors.delete_vector_bucket(vectorBucketName=bucket) +def assert_engine_wrote_like_the_gateway(cluster, index): + """The engine writes the table's files itself: the filer holds entries the + gateway could have written (an ETag, chunks, no inline content), and under + VECTOR_ENCRYPT_VOLUMES every chunk carries a cipher key.""" + import os + import urllib.request + + table = f"/buckets/{index['vectorBucketName']}/vector/{index['indexName']}" + listing = urllib.request.Request(f"http://127.0.0.1:{cluster['filer']}{table}/data/", headers={"Accept": "application/json"}) + with urllib.request.urlopen(listing, timeout=5) as response: + entries = json.load(response)["Entries"] or [] + assert entries, "the engine wrote no data file" + for entry in entries: + with urllib.request.urlopen(f"http://127.0.0.1:{cluster['filer']}{table}/data/{entry['FullPath'].rsplit('/', 1)[1]}?metadata=true", timeout=5) as response: + meta = json.load(response) + assert meta["chunks"], meta + assert "Seaweed-X-Amz-ETag" in (meta.get("Extended") or {}), meta.get("Extended") + if os.environ.get("VECTOR_ENCRYPT_VOLUMES"): + assert all(chunk.get("cipher_key") for chunk in meta["chunks"]), meta["chunks"] + + +def run_cli(cluster, *args): + """weed-vector-cli's JSON answer. It reaches the cluster through the + namespace that names the tables, the masters whose filers hold them and the + cluster token's key, and finds security.toml and the read key where the + weed processes do.""" + import os + import subprocess + from pathlib import Path + cli = Path(os.environ.get("VECTOR_CLI_BINARY", Path(__file__).resolve().parent / "weed-vector-cli")).resolve() + command = [str(cli), "--namespace", cluster["catalog"], "--masters", cluster["masters"], + "--filer-signing-key", cluster["filer_signing_key"], *args] + return json.loads(subprocess.run(command, check=True, capture_output=True, text=True, + cwd=cluster["directory"], env=cluster["env"]).stdout) + + def expect_error(code, operation, **kwargs): with pytest.raises(ClientError) as exc: operation(**kwargs) @@ -140,10 +176,11 @@ def test_empty_index(vectors, index): queryVector={"float32": [1, 0, 0]}, topK=3) -def test_upsert_and_invalid_batch(vectors, index): +def test_upsert_and_invalid_batch(vectors, index, cluster): vectors.put_vectors(**index, vectors=[{"key": "a", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"revision": 1}}]) vectors.put_vectors(**index, vectors=[{"key": "a", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"revision": 2}}]) expected = [{"key": "a", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"revision": 2}}] + assert_engine_wrote_like_the_gateway(cluster, index) for invalid in ( [{"key": "a", "data": {"float32": [1, 1, 1, 1]}}, {"key": "bad", "data": {"float32": [1, 2]}}], [{"key": "a", "data": {"float32": [1, 1, 1, 1]}}, {"key": "a", "data": {"float32": [2, 2, 2, 2]}}], @@ -232,6 +269,56 @@ def test_euclidean_distance(vectors, index): assert result["vectors"][0]["distance"] == pytest.approx(5.0) +def test_sse_s3_bucket(vectors, cluster): + """An index in a bucket that encrypts with SSE-S3 is written encrypted by + the engine, under keys the gateway can unwrap: the entries carry the + SSE-S3 attributes, the chunks hold ciphertext, and the gateway's own S3 API + serves the files decrypted.""" + import urllib.request + + bucket = "sealed-" + uuid.uuid4().hex[:16] + vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration={"sseType": "AES256"}) + args = {"vectorBucketName": bucket, "indexName": INDEX} + # A small index, whose data file is one chunk, and a wide one whose data + # file spans several: the gateway reads the two differently. + vectors.create_index(**args, dataType="float32", dimension=2, distanceMetric="euclidean") + vectors.put_vectors(**args, vectors=[{"key": "a", "data": {"float32": [1, 0]}}, + {"key": "b", "data": {"float32": [0, 1]}}]) + hits = vectors.query_vectors(**args, queryVector={"float32": [0, 1]}, topK=1)["vectors"] + assert [h["key"] for h in hits] == ["b"] + wide = {"vectorBucketName": bucket, "indexName": "wide"} + vectors.create_index(**wide, dataType="float32", dimension=4096, distanceMetric="euclidean") + vectors.put_vectors(**wide, vectors=[{"key": f"w{i}", "data": {"float32": [float(i)] * 4096}} for i in range(300)]) + hits = vectors.query_vectors(**wide, queryVector={"float32": [7.0] * 4096}, topK=1)["vectors"] + assert [h["key"] for h in hits] == ["w7"] + + s3 = boto3.client("s3", endpoint_url=cluster["endpoint"], aws_access_key_id="test", aws_secret_access_key="test", + region_name="us-east-1", config=Config(s3={"addressing_style": "path"})) + chunk_counts = set() + for index in (INDEX, "wide"): + table = f"/buckets/{bucket}/vector/{index}" + listing = urllib.request.Request(f"http://127.0.0.1:{cluster['filer']}{table}/data/", headers={"Accept": "application/json"}) + with urllib.request.urlopen(listing, timeout=5) as response: + names = [e["FullPath"].rsplit("/", 1)[1] for e in json.load(response)["Entries"] or []] + assert names, f"the engine wrote no data file for {index}" + for name in names: + with urllib.request.urlopen(f"http://127.0.0.1:{cluster['filer']}{table}/data/{name}?metadata=true", timeout=5) as response: + meta = json.load(response) + assert "x-seaweedfs-sse-s3-key" in meta["Extended"], meta["Extended"].keys() + assert all(chunk.get("sse_type") == 3 for chunk in meta["chunks"]), meta["chunks"] + chunk_counts.add(len(meta["chunks"])) + # The gateway's own S3 API serves the file decrypted: a Lance + # file, its magic in the footer, of the size the filer records. + through_gateway = s3.get_object(Bucket=bucket, Key=f"vector/{index}/data/{name}")["Body"].read() + assert through_gateway[-4:] == b"LANC", (index, name, through_gateway[-16:]) + assert len(through_gateway) == meta["FileSize"], (len(through_gateway), meta["FileSize"]) + assert 1 in chunk_counts and max(chunk_counts) > 1, chunk_counts + + vectors.delete_index(**wide) + vectors.delete_index(**args) + vectors.delete_vector_bucket(vectorBucketName=bucket) + + def test_cosine_metric(vectors): bucket = "cosine-" + uuid.uuid4().hex[:16] vectors.create_vector_bucket(vectorBucketName=bucket) @@ -514,11 +601,8 @@ def test_vector_index_job(vectors, cluster): """The maintenance worker indexes a table past the row threshold through the admin server, queries then run through the index, and rows written after the build are still found.""" - import os import random - import subprocess import urllib.request - from pathlib import Path bucket, index, dim = "vectors-indexed", "idx", 16 vectors.create_vector_bucket(vectorBucketName=bucket) @@ -555,11 +639,13 @@ def test_vector_index_job(vectors, cluster): run = call("/run", "POST", {"timeout_seconds": 110}) assert run["success_count"] == 1 and run["error_count"] == 0, run - cli = Path(os.environ.get("VECTOR_CLI_BINARY", Path(__file__).resolve().parent / "weed-vector-cli")) - coverage = json.loads(subprocess.run( - [str(cli), "--namespace", cluster["catalog"], "--access-key", "test", "--secret-key", "test", - "coverage", f"{bucket}$vector${index}"], check=True, capture_output=True, text=True).stdout) + coverage = run_cli(cluster, "coverage", f"{bucket}$vector${index}") assert len(coverage) == 1 and coverage[0]["segments"] == 1 and coverage[0]["unindexed_rows"] == 0, coverage + # A table named by its s3:// location is opened on the cluster as well. + uri = run_cli(cluster, "describe", f"{bucket}$vector${index}")["uri"] + assert uri.startswith("s3://"), uri + direct = run_cli(cluster, "coverage", uri) + assert direct == coverage, (direct, coverage) found = 0 for _ in range(20): @@ -579,9 +665,7 @@ def test_vector_index_job(vectors, cluster): assert again["detected_count"] == 0, again def coverage_now(): - return json.loads(subprocess.run( - [str(cli), "--namespace", cluster["catalog"], "--access-key", "test", "--secret-key", "test", - "coverage", f"{bucket}$vector${index}"], check=True, capture_output=True, text=True).stdout)[0] + return run_cli(cluster, "coverage", f"{bucket}$vector${index}")[0] def optimize(expect_detected): request = urllib.request.Request( @@ -654,7 +738,7 @@ def test_query_without_writer(vectors, index, cluster): def test_engine_without_credentials(open_cluster): - """With no S3 identities, an engine given no credentials still serves an index.""" + """A gateway with no S3 identities serves an index like one with.""" client = boto3.client("s3vectors", endpoint_url=open_cluster["endpoint"], aws_access_key_id="any", aws_secret_access_key="any", region_name="us-east-1", config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0})) diff --git a/weed/pb/vector.proto b/weed/pb/vector.proto index 31ed317330..466993f010 100644 --- a/weed/pb/vector.proto +++ b/weed/pb/vector.proto @@ -35,8 +35,11 @@ message Table { uint64 version = 2; // The table's Lance location, as the catalog describes it. string location = 3; - // object_store options from describe_table (vended creds / endpoint). - map storage_options = 4; + reserved 4; + reserved "storage_options"; + // The bucket encrypts its objects with SSE-S3; the engine writes the + // table's files that way. + bool sse_s3 = 5; } message CreateIndexRequest { diff --git a/weed/pb/vector_pb/vector.pb.go b/weed/pb/vector_pb/vector.pb.go index 263770c4aa..b67556cd4c 100644 --- a/weed/pb/vector_pb/vector.pb.go +++ b/weed/pb/vector_pb/vector.pb.go @@ -84,10 +84,11 @@ type Table struct { Version uint64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` // The table's Lance location, as the catalog describes it. Location string `protobuf:"bytes,3,opt,name=location,proto3" json:"location,omitempty"` - // object_store options from describe_table (vended creds / endpoint). - StorageOptions map[string]string `protobuf:"bytes,4,rep,name=storage_options,json=storageOptions,proto3" json:"storage_options,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The bucket encrypts its objects with SSE-S3; the engine writes the + // table's files that way. + SseS3 bool `protobuf:"varint,5,opt,name=sse_s3,json=sseS3,proto3" json:"sse_s3,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Table) Reset() { @@ -141,11 +142,11 @@ func (x *Table) GetLocation() string { return "" } -func (x *Table) GetStorageOptions() map[string]string { +func (x *Table) GetSseS3() bool { if x != nil { - return x.StorageOptions + return x.SseS3 } - return nil + return false } type CreateIndexRequest struct { @@ -1080,15 +1081,12 @@ var File_vector_proto protoreflect.FileDescriptor const file_vector_proto_rawDesc = "" + "\n" + - "\fvector.proto\x12\tvector_pb\x1a\x1cgoogle/protobuf/struct.proto\"\xdf\x01\n" + + "\fvector.proto\x12\tvector_pb\x1a\x1cgoogle/protobuf/struct.proto\"{\n" + "\x05Table\x12\x0e\n" + "\x02id\x18\x01 \x03(\tR\x02id\x12\x18\n" + "\aversion\x18\x02 \x01(\x04R\aversion\x12\x1a\n" + - "\blocation\x18\x03 \x01(\tR\blocation\x12M\n" + - "\x0fstorage_options\x18\x04 \x03(\v2$.vector_pb.Table.StorageOptionsEntryR\x0estorageOptions\x1aA\n" + - "\x13StorageOptionsEntry\x12\x10\n" + - "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xdf\x01\n" + + "\blocation\x18\x03 \x01(\tR\blocation\x12\x15\n" + + "\x06sse_s3\x18\x05 \x01(\bR\x05sseS3J\x04\b\x04\x10\x05R\x0fstorage_options\"\xdf\x01\n" + "\x12CreateIndexRequest\x12&\n" + "\x05table\x18\x01 \x01(\v2\x10.vector_pb.TableR\x05table\x12\x1c\n" + "\tdimension\x18\x02 \x01(\rR\tdimension\x12B\n" + @@ -1186,7 +1184,7 @@ func file_vector_proto_rawDescGZIP() []byte { } var file_vector_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_vector_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_vector_proto_msgTypes = make([]protoimpl.MessageInfo, 17) var file_vector_proto_goTypes = []any{ (DistanceMetric)(0), // 0: vector_pb.DistanceMetric (*Table)(nil), // 1: vector_pb.Table @@ -1206,44 +1204,42 @@ var file_vector_proto_goTypes = []any{ (*QueryVectorsRequest)(nil), // 15: vector_pb.QueryVectorsRequest (*QueryResult)(nil), // 16: vector_pb.QueryResult (*QueryVectorsResponse)(nil), // 17: vector_pb.QueryVectorsResponse - nil, // 18: vector_pb.Table.StorageOptionsEntry - (*structpb.Struct)(nil), // 19: google.protobuf.Struct + (*structpb.Struct)(nil), // 18: google.protobuf.Struct } var file_vector_proto_depIdxs = []int32{ - 18, // 0: vector_pb.Table.storage_options:type_name -> vector_pb.Table.StorageOptionsEntry - 1, // 1: vector_pb.CreateIndexRequest.table:type_name -> vector_pb.Table - 0, // 2: vector_pb.CreateIndexRequest.distance_metric:type_name -> vector_pb.DistanceMetric - 1, // 3: vector_pb.DropIndexRequest.table:type_name -> vector_pb.Table - 19, // 4: vector_pb.Vector.metadata:type_name -> google.protobuf.Struct - 1, // 5: vector_pb.PutVectorsRequest.table:type_name -> vector_pb.Table - 6, // 6: vector_pb.PutVectorsRequest.vectors:type_name -> vector_pb.Vector - 1, // 7: vector_pb.GetVectorsRequest.table:type_name -> vector_pb.Table - 6, // 8: vector_pb.GetVectorsResponse.vectors:type_name -> vector_pb.Vector - 1, // 9: vector_pb.DeleteVectorsRequest.table:type_name -> vector_pb.Table - 1, // 10: vector_pb.ListVectorsRequest.table:type_name -> vector_pb.Table - 6, // 11: vector_pb.ListVectorsResponse.vectors:type_name -> vector_pb.Vector - 1, // 12: vector_pb.QueryVectorsRequest.table:type_name -> vector_pb.Table - 19, // 13: vector_pb.QueryResult.metadata:type_name -> google.protobuf.Struct - 16, // 14: vector_pb.QueryVectorsResponse.vectors:type_name -> vector_pb.QueryResult - 2, // 15: vector_pb.VectorService.CreateIndex:input_type -> vector_pb.CreateIndexRequest - 4, // 16: vector_pb.VectorService.DropIndex:input_type -> vector_pb.DropIndexRequest - 7, // 17: vector_pb.VectorService.PutVectors:input_type -> vector_pb.PutVectorsRequest - 9, // 18: vector_pb.VectorService.GetVectors:input_type -> vector_pb.GetVectorsRequest - 11, // 19: vector_pb.VectorService.DeleteVectors:input_type -> vector_pb.DeleteVectorsRequest - 13, // 20: vector_pb.VectorService.ListVectors:input_type -> vector_pb.ListVectorsRequest - 15, // 21: vector_pb.VectorService.QueryVectors:input_type -> vector_pb.QueryVectorsRequest - 3, // 22: vector_pb.VectorService.CreateIndex:output_type -> vector_pb.CreateIndexResponse - 5, // 23: vector_pb.VectorService.DropIndex:output_type -> vector_pb.DropIndexResponse - 8, // 24: vector_pb.VectorService.PutVectors:output_type -> vector_pb.PutVectorsResponse - 10, // 25: vector_pb.VectorService.GetVectors:output_type -> vector_pb.GetVectorsResponse - 12, // 26: vector_pb.VectorService.DeleteVectors:output_type -> vector_pb.DeleteVectorsResponse - 14, // 27: vector_pb.VectorService.ListVectors:output_type -> vector_pb.ListVectorsResponse - 17, // 28: vector_pb.VectorService.QueryVectors:output_type -> vector_pb.QueryVectorsResponse - 22, // [22:29] is the sub-list for method output_type - 15, // [15:22] is the sub-list for method input_type - 15, // [15:15] is the sub-list for extension type_name - 15, // [15:15] is the sub-list for extension extendee - 0, // [0:15] is the sub-list for field type_name + 1, // 0: vector_pb.CreateIndexRequest.table:type_name -> vector_pb.Table + 0, // 1: vector_pb.CreateIndexRequest.distance_metric:type_name -> vector_pb.DistanceMetric + 1, // 2: vector_pb.DropIndexRequest.table:type_name -> vector_pb.Table + 18, // 3: vector_pb.Vector.metadata:type_name -> google.protobuf.Struct + 1, // 4: vector_pb.PutVectorsRequest.table:type_name -> vector_pb.Table + 6, // 5: vector_pb.PutVectorsRequest.vectors:type_name -> vector_pb.Vector + 1, // 6: vector_pb.GetVectorsRequest.table:type_name -> vector_pb.Table + 6, // 7: vector_pb.GetVectorsResponse.vectors:type_name -> vector_pb.Vector + 1, // 8: vector_pb.DeleteVectorsRequest.table:type_name -> vector_pb.Table + 1, // 9: vector_pb.ListVectorsRequest.table:type_name -> vector_pb.Table + 6, // 10: vector_pb.ListVectorsResponse.vectors:type_name -> vector_pb.Vector + 1, // 11: vector_pb.QueryVectorsRequest.table:type_name -> vector_pb.Table + 18, // 12: vector_pb.QueryResult.metadata:type_name -> google.protobuf.Struct + 16, // 13: vector_pb.QueryVectorsResponse.vectors:type_name -> vector_pb.QueryResult + 2, // 14: vector_pb.VectorService.CreateIndex:input_type -> vector_pb.CreateIndexRequest + 4, // 15: vector_pb.VectorService.DropIndex:input_type -> vector_pb.DropIndexRequest + 7, // 16: vector_pb.VectorService.PutVectors:input_type -> vector_pb.PutVectorsRequest + 9, // 17: vector_pb.VectorService.GetVectors:input_type -> vector_pb.GetVectorsRequest + 11, // 18: vector_pb.VectorService.DeleteVectors:input_type -> vector_pb.DeleteVectorsRequest + 13, // 19: vector_pb.VectorService.ListVectors:input_type -> vector_pb.ListVectorsRequest + 15, // 20: vector_pb.VectorService.QueryVectors:input_type -> vector_pb.QueryVectorsRequest + 3, // 21: vector_pb.VectorService.CreateIndex:output_type -> vector_pb.CreateIndexResponse + 5, // 22: vector_pb.VectorService.DropIndex:output_type -> vector_pb.DropIndexResponse + 8, // 23: vector_pb.VectorService.PutVectors:output_type -> vector_pb.PutVectorsResponse + 10, // 24: vector_pb.VectorService.GetVectors:output_type -> vector_pb.GetVectorsResponse + 12, // 25: vector_pb.VectorService.DeleteVectors:output_type -> vector_pb.DeleteVectorsResponse + 14, // 26: vector_pb.VectorService.ListVectors:output_type -> vector_pb.ListVectorsResponse + 17, // 27: vector_pb.VectorService.QueryVectors:output_type -> vector_pb.QueryVectorsResponse + 21, // [21:28] is the sub-list for method output_type + 14, // [14:21] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_vector_proto_init() } @@ -1257,7 +1253,7 @@ func file_vector_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_vector_proto_rawDesc), len(file_vector_proto_rawDesc)), NumEnums: 1, - NumMessages: 18, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/s3api/lance/cluster_token_test.go b/weed/s3api/lance/cluster_token_test.go index 15409a3aa3..71340bec92 100644 --- a/weed/s3api/lance/cluster_token_test.go +++ b/weed/s3api/lance/cluster_token_test.go @@ -4,10 +4,14 @@ import ( "encoding/hex" "net/http" "net/http/httptest" + "strings" "testing" "time" jwt "github.com/golang-jwt/jwt/v5" + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "github.com/seaweedfs/seaweedfs/weed/security" ) @@ -85,6 +89,84 @@ func TestClusterTokenMarksTheCallerTrusted(t *testing.T) { } } +// refusingAuthenticator turns every caller away, as the S3 authenticator does +// for a request that signs with nothing. +type refusingAuthenticator struct{} + +func (refusingAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) { + return "", nil, s3err.ErrAccessDenied +} + +func (refusingAuthenticator) DefaultAllow() bool { return false } + +func (refusingAuthenticator) AuthEnabled() bool { return true } + +// A request carrying the cluster token alone is a cluster process: it is +// authenticated by the token, runs through the trusted manager, and is vended +// no credentials. The same request without the token is turned away. +func TestClusterTokenAloneAuthenticatesAClusterProcess(t *testing.T) { + h := newTestHarness(t) + h.createBucket(t, "lnc", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/namespace/lnc$ns/create", `{}`, http.StatusOK) + h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$t/declare", `{}`, http.StatusOK) + + key := security.SigningKey("cluster-key") + server := NewServer(s3tables.NewManagerClient(h.filer.Client), refusingAuthenticator{}) + server.SetSigningKey(func() security.SigningKey { return key }) + server.SetCredentialVendor(recordingVendor{}) + router := mux.NewRouter().SkipClean(true) + server.RegisterRoutes(router) + token := string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60)) + + describe := func(withToken bool) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/v1/table/lnc$ns$t/describe", strings.NewReader(`{"vend_credentials":true}`)) + if withToken { + req.Header.Set(clusterTokenHeader, token) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec + } + if rec := describe(false); rec.Code == http.StatusOK { + t.Fatalf("a request without a credential described the table: %s", rec.Body.String()) + } + rec := describe(true) + if rec.Code != http.StatusOK { + t.Fatalf("describe with the cluster token = %d (%s)", rec.Code, rec.Body.String()) + } + described := decode[DescribeTableResponse](t, rec) + if described.Location != "s3://lnc/ns/t" { + t.Fatalf("location = %q", described.Location) + } + if _, vended := described.StorageOptions["aws_access_key_id"]; vended { + t.Fatalf("a cluster process was vended credentials: %v", described.StorageOptions) + } + + list := httptest.NewRequest(http.MethodGet, "/v1/namespace/lnc$ns/table/list", nil) + list.Header.Set(clusterTokenHeader, token) + rec = httptest.NewRecorder() + router.ServeHTTP(rec, list) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "lnc$ns$t") { + t.Fatalf("list with the cluster token = %d (%s)", rec.Code, rec.Body.String()) + } + + // The vector namespace, read-only to every other Lance caller, takes a + // cluster process's register and deregister. + h.seedVectorIndex(t, "vec", s3tables.FormatLance) + for _, tc := range []struct{ target, body string }{ + {"/v1/table/vec$vector$idx/register", `{"location":"s3://vec/vector/idx","mode":"overwrite"}`}, + {"/v1/table/vec$vector$idx/deregister", `{}`}, + } { + req := httptest.NewRequest(http.MethodPost, tc.target, strings.NewReader(tc.body)) + req.Header.Set(clusterTokenHeader, token) + rec = httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s with the cluster token = %d (%s)", tc.target, rec.Code, rec.Body.String()) + } + } +} + // The Rust catalog client derives the same key; its tests pin this value too. func TestClusterTokenKeyDerivation(t *testing.T) { got := hex.EncodeToString(clusterTokenKey(security.SigningKey("cluster-key"))) diff --git a/weed/s3api/lance/handlers_table.go b/weed/s3api/lance/handlers_table.go index 11d59ca81f..417a734bca 100644 --- a/weed/s3api/lance/handlers_table.go +++ b/weed/s3api/lance/handlers_table.go @@ -399,7 +399,7 @@ func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) { writeError(w, r, http.StatusConflict, codeTableAlreadyExists, "table already exists") return } - if err := s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns); err != nil { + if err := s.manager(r).CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns); err != nil { writeStorageError(w, r, err) return } @@ -458,7 +458,7 @@ func (s *Server) handleDeregisterTable(w http.ResponseWriter, r *http.Request) { writeStorageError(w, r, err) return } - if err := s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns); err != nil { + if err := s.manager(r).CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns); err != nil { writeStorageError(w, r, err) return } diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index 9dd33c99a1..7c97fff291 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -192,11 +192,18 @@ func (s *Server) authEnabled() bool { // Auth authenticates the caller and puts the identity in the request context. // Lance clients authenticate the catalog with a Bearer token or an x-api-key -// header; the S3 authenticator stays for callers that can SigV4-sign. +// header; the S3 authenticator stays for callers that can SigV4-sign. A +// cluster process (the vector engine, the maintenance worker) presents the +// cluster token alone, which is its credential: it holds the cluster's filer +// signing key, and the catalog is read on its behalf by the trusted manager. func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if s.hasClusterToken(r) { r = r.WithContext(withTrustedCaller(r.Context())) + if r.Header.Get("Authorization") == "" && r.Header.Get("x-api-key") == "" { + handler(w, r) + return + } } // A request carrying a Bearer token is a Lance REST client. An invalid // or expired token gets 401 immediately; falling through to the S3 diff --git a/weed/s3api/lance/storage.go b/weed/s3api/lance/storage.go index fb5bfaa50f..3b8f9076ca 100644 --- a/weed/s3api/lance/storage.go +++ b/weed/s3api/lance/storage.go @@ -17,11 +17,21 @@ import ( const defaultPageSize = 1000 +// manager is the S3 Tables manager a request runs through: a cluster process, +// authenticated by the cluster token alone, has no identity of its own and is +// trusted; every other caller is checked against its grants. +func (s *Server) manager(r *http.Request) *s3tables.Manager { + if s3_constants.GetIdentityNameFromContext(r) == "" && isTrustedCaller(r.Context()) { + return s.internal + } + return s.tablesManager +} + // execute runs one S3 Tables operation as the request's authenticated caller. func (s *Server) execute(r *http.Request, operation string, req, resp interface{}) error { identityName := s3_constants.GetIdentityNameFromContext(r) return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return s.tablesManager.Execute(r.Context(), s3tables.NewManagerClient(client), operation, req, resp, identityName) + return s.manager(r).Execute(r.Context(), s3tables.NewManagerClient(client), operation, req, resp, identityName) }) } @@ -169,6 +179,11 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b } principal := s3_constants.GetIdentityNameFromContext(r) + // A cluster process reaches the table's files on the filers and volume + // servers itself; it has no principal and needs no S3 credentials. + if principal == "" && isTrustedCaller(r.Context()) { + return options, nil + } credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location), canWrite == nil || !canWrite()) if err != nil { return nil, err diff --git a/weed/s3api/lance/vector_namespace_test.go b/weed/s3api/lance/vector_namespace_test.go index 3185684df3..f62e52c137 100644 --- a/weed/s3api/lance/vector_namespace_test.go +++ b/weed/s3api/lance/vector_namespace_test.go @@ -106,7 +106,9 @@ func (h *testHarness) vendedKey(t *testing.T, target, token string) string { } // A caller holding every s3tables grant still gets read-only credentials to a -// vector bucket's index data; only the cluster's own components can write it. +// vector bucket's index data; only the cluster's own components can write it, +// and a cluster process presenting the token alone reaches the files itself, +// so it is vended nothing. func TestVectorNamespaceVendsWriteCredentialsOnlyToTrustedCallers(t *testing.T) { h := newTestHarness(t) key := security.SigningKey("cluster-key") @@ -129,8 +131,8 @@ func TestVectorNamespaceVendsWriteCredentialsOnlyToTrustedCallers(t *testing.T) {"describe the index", "/v1/table/vec$vector$idx/describe", "", "read-only"}, {"declare the index", "/v1/table/vec$vector$idx/declare", "", "read-only"}, {"describe with an admin token", "/v1/table/vec$vector$idx/describe", forged, "read-only"}, - {"trusted describe", "/v1/table/vec$vector$idx/describe", trusted, "read-write"}, - {"trusted declare", "/v1/table/vec$vector$idx/declare", trusted, "read-write"}, + {"trusted describe", "/v1/table/vec$vector$idx/describe", trusted, ""}, + {"trusted declare", "/v1/table/vec$vector$idx/declare", trusted, ""}, {"another namespace", "/v1/table/vec$other$tbl/describe", "", "read-write"}, {"bucket from before formats", "/v1/table/old$vector$idx/describe", "", "read-write"}, } { diff --git a/weed/s3api/s3tables/utils.go b/weed/s3api/s3tables/utils.go index 7837ed9cbc..411505773c 100644 --- a/weed/s3api/s3tables/utils.go +++ b/weed/s3api/s3tables/utils.go @@ -202,10 +202,19 @@ func DeclaredBucketEncryption(entry *filer_pb.Entry) string { return "" } var metadata tableBucketMetadata - if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil || !IsCatalogOnlyFormat(metadata.Format) { + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil { return "" } - if algorithm, _ := metadata.EncryptionConfiguration["sseAlgorithm"].(string); algorithm == "AES256" { + return DeclaredEncryption(metadata.Format, metadata.EncryptionConfiguration) +} + +// DeclaredEncryption is DeclaredBucketEncryption for a bucket record already +// read: its format and encryption configuration. +func DeclaredEncryption(format string, encryption map[string]interface{}) string { + if !IsCatalogOnlyFormat(format) { + return "" + } + if algorithm, _ := encryption["sseAlgorithm"].(string); algorithm == "AES256" { return algorithm } return "" diff --git a/weed/s3api/s3vectors/describe.go b/weed/s3api/s3vectors/describe.go index b54d4657db..a9bb325174 100644 --- a/weed/s3api/s3vectors/describe.go +++ b/weed/s3api/s3vectors/describe.go @@ -20,25 +20,41 @@ type DescribeFn func(ctx context.Context, id []string) (string, map[string]strin // SetDescribe resolves every table this server sends to the engine, so the // engine opens it without a catalog round trip of its own. func (s *Server) SetDescribe(describe DescribeFn) { - s.vectors = &describedClient{VectorServiceClient: s.vectors, describe: describe} + s.vectors = &describedClient{VectorServiceClient: s.vectors, describe: describe, encrypts: s.bucketEncrypts} } -// describedClient fills in the table's location and storage options before a -// request reaches the engine. +// bucketEncrypts reports whether the vector bucket declares SSE-S3, which the +// S3 write path applies to every file of a Lance bucket and the engine must +// apply to the files it writes itself. +func (s *Server) bucketEncrypts(ctx context.Context, bucket string) (bool, error) { + record, err := s.vectorBucket(ctx, bucket) + if err != nil { + return false, err + } + return s3tables.DeclaredEncryption(record.Format, record.EncryptionConfiguration) != "", nil +} + +// describedClient fills in the table's location and the bucket's encryption +// before a request reaches the engine. type describedClient struct { vector_pb.VectorServiceClient describe DescribeFn + encrypts func(ctx context.Context, bucket string) (bool, error) } func (c *describedClient) resolve(ctx context.Context, table *vector_pb.Table) error { - if table == nil || table.Location != "" { + if table == nil || table.Location != "" || len(table.Id) == 0 { return nil } - location, options, err := c.describe(ctx, table.Id) + location, _, err := c.describe(ctx, table.Id) if err != nil { return describeErr(err) } - table.Location, table.StorageOptions = location, options + sseS3, err := c.encrypts(ctx, table.Id[0]) + if err != nil { + return describeErr(err) + } + table.Location, table.SseS3 = location, sseS3 return nil } diff --git a/weed/s3api/s3vectors/describe_test.go b/weed/s3api/s3vectors/describe_test.go index 2a94e4bb80..3b9d79b196 100644 --- a/weed/s3api/s3vectors/describe_test.go +++ b/weed/s3api/s3vectors/describe_test.go @@ -49,14 +49,43 @@ func TestDescribeFillsTableLocation(t *testing.T) { t.Fatalf("table location = %q", table.GetLocation()) } } - if got := engine.puts[0].Table.StorageOptions["aws_endpoint"]; got != "http://gateway:8333" { - t.Fatalf("storage options = %v", engine.puts[0].Table.StorageOptions) + if engine.puts[0].Table.SseS3 { + t.Fatal("a bucket without encryption was sent as SSE-S3") } if len(described) != 2 || strings.Join(described[1], "/") != "vec-bucket/vector/idx" { t.Fatalf("described ids = %v", described) } } +// A bucket created with SSE-S3 sends its tables as encrypted. +func TestDescribeCarriesTheBucketsEncryption(t *testing.T) { + fs := s3tablestest.Start(t) + engine := &fakeVectors{} + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine) + s.SetEncryptionChecker(func() bool { return true }) + s.SetDescribe(func(_ context.Context, id []string) (string, map[string]string, error) { + return "s3://" + strings.Join(id, "/"), nil, nil + }) + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{ + VectorBucketName: "sealed", + EncryptionConfiguration: map[string]interface{}{"sseType": "AES256"}, + }); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "sealed", + IndexName: "idx", + DataType: "float32", + Dimension: 3, + DistanceMetric: "euclidean", + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + if len(engine.created) != 1 || !engine.created[0].Table.SseS3 { + t.Fatalf("engine CreateIndex calls = %v, want one SSE-S3 table", engine.created) + } +} + func TestDescribeErrorsReachTheCaller(t *testing.T) { fs := s3tablestest.Start(t) cases := []struct {