From b650303aee7ce4e41fa9a0e5346fb38de09a4127 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 7 Nov 2026 04:21:13 -0800 Subject: [PATCH] S3 Vectors: weed-vector reads tables from the filers and volume servers (#12469) * sw-client: the filer and volume client library The client the Enterprise repository's VFS and vector engine use, with the same API: filer gRPC for entries, listings, volume locations, assignments and remote caching; volume HTTP, plain or TLS, for needle bytes; chunk resolution for sparse, overlapping and manifest chunks; a location cache; and the write side. Enterprise-only protocol fields (license, conditional mutations, hard links, KV listing) are left out. Compiled against weed/pb/filer.proto like the other worker crates. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-lance-worker: reads straight from the filers and volume servers direct_read::DirectReads wraps the object store Lance opens so that entries, listings and bytes come from the filers (gRPC) and the volume servers (HTTP, signed with the read JWT), whatever the entry holds: inline content, chunk manifests, sparse or overlapping chunks, the filer's volume encryption, the gateway's SSE-S3 (sse.rs, with the KEK from security.toml or the filer) and objects tiered to remote storage. Writes still go to the inner store. The code and its tests are those of the Enterprise repository's engine. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-vector: read tables from the filers and volume servers The engine asks the masters for the filers (and follows the list), takes jwt.signing.read.key, grpc.client, https.client and the SSE-S3 keys from security.toml, and opens every table with DirectReads innermost and the read cache outside it. --masters is required: the gateway finds the engine through them and the engine finds the filers. Writes still go to the gateway. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * sse: chunks out of a manifest keep their own SSE-S3 keys A chunk expanded from a manifest carries its id as fid, so keying the per-chunk key cache by file_id gave every such chunk the first one's key and IV. Keyed by the chunk's id however it is carried. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * direct reads: a silent filer is passed over like an unreachable one Each filer attempt is bounded, so a filer that accepts the connection and never answers does not hold the read or the startup lookup. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * sw-client: carry the assignment's fsync to the volume upload The filer's storage rule may want a needle on disk before the volume answers; the assignment says so and the upload now asks for it, as the Go uploader does. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-vector: argument tests pass the required masters; connect timeout for filers Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * direct reads: a failed read is tried once more against fresh locations A fetch that fails forgets the volume locations it was made against; the request now looks them up again and retries instead of failing on a volume that moved. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * direct reads: the filer deadline bounds a listing page, not the listing Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-vector: security.toml booleans read as viper reads them Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-vector: a filer the master lists badly does not hide the others Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-vector: register with the masters only once ready to serve Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * sw-client: the HTTP transport asks the volume to fsync when the assignment does write_synced fell back to the trait default, which refuses fsync, so every upload under a fsync storage rule failed before it was sent. Co-Authored-By: Claude Opus 5.5 * sw-client: https.client.ca alone vouches for volume servers With a CA configured, reqwest still trusted the system roots next to it; weed processes trust only the configured pool. Co-Authored-By: Claude Opus 5.5 * sw-client: an empty volume range is read without a request A zero length underflowed the Range end, and a range past u64::MAX overflowed it; the first now returns no bytes, the second an error. Co-Authored-By: Claude Opus 5.5 * sw-client: a whole-needle answer to a ranged read is cut to the range A server or proxy that ignores Range answers 200 with the whole needle; the read returned its first bytes in place of the ones asked for. Co-Authored-By: Claude Opus 5.5 * sw-client: a read size past u64::MAX reads to the end of the file offset + size overflowed before the clamp to the file size. Co-Authored-By: Claude Opus 5.5 * sw-client: an SSE-encrypted manifest is refused without a decoder Only filer-encrypted manifests were; an SSE one went to the protobuf decoder as ciphertext. Co-Authored-By: Claude Opus 5.5 * sw-client: an empty write leaves the staged file size alone A zero-byte write past the end grew the size to its offset, as a zero-byte pwrite does not. Co-Authored-By: Claude Opus 5.5 * sw-client: the chunk cache keeps no empty chunks They add no bytes, so nothing evicted them and the map grew with every distinct empty needle, even at capacity zero. Co-Authored-By: Claude Opus 5.5 * sw-client: flushing with a zero chunk size is an error, not a panic Co-Authored-By: Claude Opus 5.5 * weed-vector: an empty master list fails at startup "" split into one empty address that passed the check, and the engine then waited two minutes for a filer before a generic error. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 --- seaweed-vector/Cargo.lock | 589 +++++- seaweed-vector/crates/server/Cargo.toml | 1 + seaweed-vector/crates/server/src/main.rs | 176 +- .../crates/server/src/master_client.rs | 75 +- seaweed-vector/crates/server/src/security.rs | 48 +- seaweed-vector/crates/server/src/service.rs | 29 +- seaweed-worker/Cargo.lock | 599 +++++- seaweed-worker/Cargo.toml | 2 +- seaweed-worker/crates/lance/Cargo.toml | 20 + .../crates/lance/src/direct_read.rs | 1687 +++++++++++++++++ seaweed-worker/crates/lance/src/lib.rs | 2 + seaweed-worker/crates/lance/src/sse.rs | 776 ++++++++ .../lance/tests/direct_read_fixture.json | 26 + seaweed-worker/crates/sw-client/Cargo.toml | 34 + seaweed-worker/crates/sw-client/build.rs | 24 + seaweed-worker/crates/sw-client/src/cache.rs | 129 ++ seaweed-worker/crates/sw-client/src/filer.rs | 1287 +++++++++++++ seaweed-worker/crates/sw-client/src/lib.rs | 45 + .../crates/sw-client/src/location_cache.rs | 303 +++ seaweed-worker/crates/sw-client/src/read.rs | 1509 +++++++++++++++ .../crates/sw-client/src/transport.rs | 633 +++++++ seaweed-worker/crates/sw-client/src/wbuf.rs | 348 ++++ seaweed-worker/crates/sw-client/src/write.rs | 566 ++++++ 23 files changed, 8862 insertions(+), 46 deletions(-) create mode 100644 seaweed-worker/crates/lance/src/direct_read.rs create mode 100644 seaweed-worker/crates/lance/src/sse.rs create mode 100644 seaweed-worker/crates/lance/tests/direct_read_fixture.json create mode 100644 seaweed-worker/crates/sw-client/Cargo.toml create mode 100644 seaweed-worker/crates/sw-client/build.rs create mode 100644 seaweed-worker/crates/sw-client/src/cache.rs create mode 100644 seaweed-worker/crates/sw-client/src/filer.rs create mode 100644 seaweed-worker/crates/sw-client/src/lib.rs create mode 100644 seaweed-worker/crates/sw-client/src/location_cache.rs create mode 100644 seaweed-worker/crates/sw-client/src/read.rs create mode 100644 seaweed-worker/crates/sw-client/src/transport.rs create mode 100644 seaweed-worker/crates/sw-client/src/wbuf.rs create mode 100644 seaweed-worker/crates/sw-client/src/write.rs diff --git a/seaweed-vector/Cargo.lock b/seaweed-vector/Cargo.lock index 8590908110..d24b8df03a 100644 --- a/seaweed-vector/Cargo.lock +++ b/seaweed-vector/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -661,10 +696,10 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http", "percent-encoding", - "sha2", + "sha2 0.11.1", "time", "tracing", ] @@ -929,6 +964,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" @@ -951,6 +992,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bigdecimal" version = "0.4.11" @@ -1144,6 +1191,16 @@ dependencies = [ "phf", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.7" @@ -1251,6 +1308,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -1400,6 +1463,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -1407,6 +1482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1450,6 +1526,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.6" @@ -1459,6 +1544,33 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling" version = "0.24.1" @@ -1833,7 +1945,7 @@ dependencies = [ "num-traits", "rand 0.9.5", "regex", - "sha2", + "sha2 0.11.1", "uuid", ] @@ -2151,6 +2263,17 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "der-parser" version = "9.0.0" @@ -2181,6 +2304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -2192,7 +2316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] @@ -2250,12 +2374,71 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.42" @@ -2324,6 +2507,22 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.15" @@ -2526,6 +2725,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2567,6 +2767,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "glob" version = "0.3.4" @@ -2585,6 +2795,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.4.20" @@ -2673,6 +2894,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -3030,6 +3269,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "io-uring" version = "0.7.15" @@ -3214,6 +3462,30 @@ dependencies = [ "zmij", ] +[[package]] +name = "jsonwebtoken" +version = "10.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" +dependencies = [ + "base64 0.22.1", + "ed25519-dalek", + "getrandom 0.2.17", + "hmac 0.12.1", + "js-sys", + "p256", + "p384", + "pem", + "rand 0.8.8", + "rsa", + "serde", + "serde_json", + "sha2 0.10.9", + "signature", + "simple_asn1", + "zeroize", +] + [[package]] name = "lance" version = "10.0.0" @@ -3774,6 +4046,9 @@ name = "lazy_static" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] [[package]] name = "lexical-core" @@ -4140,6 +4415,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4164,6 +4455,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -4260,6 +4561,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "opendal" version = "0.57.0" @@ -4404,6 +4711,30 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + [[package]] name = "parking" version = "2.2.1" @@ -4451,6 +4782,25 @@ dependencies = [ "stfu8", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4519,12 +4869,45 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -4576,6 +4959,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -4852,13 +5244,24 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -4873,6 +5276,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -4883,6 +5296,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -5079,13 +5501,13 @@ dependencies = [ "base64 0.23.1", "bytes", "hex", - "hmac", + "hmac 0.13.0", "http", "jiff", "log", "percent-encoding", "sha1 0.11.0", - "sha2", + "sha2 0.11.1", "windows-sys 0.61.2", ] @@ -5184,6 +5606,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + [[package]] name = "ring" version = "0.17.14" @@ -5208,6 +5640,26 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -5446,6 +5898,20 @@ dependencies = [ "seaweed-worker-core", ] +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -5622,6 +6088,17 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.1" @@ -5658,6 +6135,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -5680,6 +6167,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.21", + "time", +] + [[package]] name = "siphasher" version = "1.0.4" @@ -5729,6 +6228,22 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlparser" version = "0.62.0" @@ -5811,6 +6326,25 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "sw-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "log", + "prost", + "protoc-bin-vendored", + "reqwest 0.12.28", + "rustls", + "thiserror 2.0.21", + "tokio", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + [[package]] name = "syn" version = "2.0.119" @@ -6347,6 +6881,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -6583,18 +7127,31 @@ dependencies = [ name = "weed-lance-worker" version = "0.1.0" dependencies = [ + "aes", + "aes-gcm", "anyhow", "arrow-cast", "arrow-schema", "async-trait", + "base64 0.22.1", + "bytes", "chrono", "clap", + "ctr", + "flate2", "futures", + "hex", + "hkdf", + "hmac 0.12.1", + "jsonwebtoken", "lance", "lance-datafusion", "lance-index", "lance-linalg", "lance-table", + "md-5 0.10.6", + "moka", + "object_store", "prometheus", "reqwest 0.12.28", "roaring", @@ -6603,9 +7160,12 @@ dependencies = [ "seaweed-worker-sort", "serde", "serde_json", + "sha2 0.10.9", + "sw-client", "tokio", "tracing", "tracing-subscriber", + "zstd", ] [[package]] @@ -6661,6 +7221,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "sw-client", "tempfile", "tokio", "tokio-rustls", @@ -6955,6 +7516,20 @@ name = "zeroize" version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/seaweed-vector/crates/server/Cargo.toml b/seaweed-vector/crates/server/Cargo.toml index 22fcaf3c21..1842651822 100644 --- a/seaweed-vector/crates/server/Cargo.toml +++ b/seaweed-vector/crates/server/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] weed-vector-core = { path = "../core" } weed-lance-worker = { path = "../../../seaweed-worker/crates/lance" } +sw-client = { path = "../../../seaweed-worker/crates/sw-client" } lance = { version = "10", default-features = false, features = ["aws"] } lance-arrow = "10" lance-linalg = "10" diff --git a/seaweed-vector/crates/server/src/main.rs b/seaweed-vector/crates/server/src/main.rs index 6b5304877b..897190c3c4 100644 --- a/seaweed-vector/crates/server/src/main.rs +++ b/seaweed-vector/crates/server/src/main.rs @@ -1,10 +1,14 @@ use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; use clap::Parser; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; use weed_lance_worker::dataset::FallbackOptions; +use weed_lance_worker::direct_read::{Connection, DirectReads, FilerDialer}; +use weed_vector_server::master_client; use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer; use weed_vector_server::read_cache::ReadCache; use weed_vector_server::security::Security; @@ -22,10 +26,18 @@ struct Args { /// only with the aws_endpoint given here. #[arg(long = "storage-option", alias = "storage.option", value_parser = parse_option)] storage_options: Vec<(String, String)>, - /// Comma-separated master addresses to register this server with; the - /// gateway discovers it via ListClusterNodes(client_type=vector). + /// Comma-separated master addresses. The engine registers with them, so + /// the gateway finds it, and asks them for the filers it reads through. #[arg(long, alias = "master", env = "WEED_VECTOR_MASTERS")] - masters: Option, + masters: String, + /// The filer group whose filers the engine reads through. + #[arg( + long, + alias = "filer.group", + env = "WEED_VECTOR_FILER_GROUP", + default_value = "" + )] + filer_group: String, /// Memory for caching reads of write-once table files, in MiB; 0 disables. #[arg(long, env = "WEED_VECTOR_READ_CACHE_MB", default_value_t = 512)] read_cache_mb: u64, @@ -85,6 +97,68 @@ fn normalize_args(args: Vec) -> Vec { /// A wildcard listen address is not reachable for peers; derive the outbound /// IP toward the first master instead — the address the master sees anyway. +/// How often the masters are asked for the filers. +const FILER_REFRESH: Duration = Duration::from_secs(30); +/// How long startup waits for a filer to be registered. +const FILER_WAIT: Duration = Duration::from_secs(120); + +/// The filers the masters list, waiting for the first to register. +async fn discover_filers( + masters: &[String], + filer_group: &str, + tls: Option<&weed_vector_server::security::GrpcTls>, +) -> Result> { + let deadline = tokio::time::Instant::now() + FILER_WAIT; + loop { + match master_client::list_filers(masters, filer_group, tls).await { + Ok(filers) if !filers.is_empty() => return Ok(filers), + Ok(_) => tracing::warn!("no filer is registered with the masters yet"), + Err(err) => tracing::warn!("listing filers: {err:#}"), + } + if tokio::time::Instant::now() >= deadline { + anyhow::bail!("no filer registered with the masters within {FILER_WAIT:?}"); + } + tokio::time::sleep(Duration::from_secs(3)).await; + } +} + +async fn follow_filers( + reads: DirectReads, + masters: Vec, + filer_group: String, + tls: Option, +) { + loop { + tokio::time::sleep(FILER_REFRESH).await; + match master_client::list_filers(&masters, &filer_group, tls.as_ref()).await { + Ok(filers) if !filers.is_empty() => { + if let Err(err) = reads.set_filers(&filers) { + tracing::warn!("filers {}: {err:#}", filers.join(",")); + } + } + // Keep the last known filers while the masters list none. + Ok(_) => tracing::warn!("the masters list no filer; keeping the known ones"), + Err(err) => tracing::warn!("listing filers: {err:#}"), + } + } +} + +fn master_addresses(masters: &str) -> Result> { + let masters: Vec = masters + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + if masters.is_empty() { + anyhow::bail!("--masters (WEED_VECTOR_MASTERS) lists no master address"); + } + for master in &masters { + master_client::grpc_addr(master)?; + } + Ok(masters) +} + fn advertised_addr(listen: &str, masters: &[String]) -> String { let Some((host, _)) = listen.rsplit_once(':') else { return listen.to_string(); @@ -129,10 +203,7 @@ async fn main() -> Result<()> { if let Some(path) = security.path() { tracing::info!("security settings from {}", path.display()); } - let master_tls = match &args.masters { - Some(_) => security.grpc_client_tls("grpc.client", "grpc.master")?, - None => None, - }; + let master_tls = security.grpc_client_tls("grpc.client", "grpc.master")?; let server_tls = security .grpc_server_tls("grpc.vector")? .map(ServerTls::load) @@ -142,21 +213,56 @@ async fn main() -> Result<()> { let listener = tokio::net::TcpListener::bind(&args.listen) .await .with_context(|| format!("listen on {}", args.listen))?; - if let Some(masters) = &args.masters { - let masters: Vec = masters.split(',').map(|s| s.trim().to_string()).collect(); - for master in &masters { - weed_vector_server::master_client::grpc_addr(master)?; - } - let advertise = advertised_addr(&args.listen, &masters); - tokio::spawn(weed_vector_server::master_client::keep_connected( - masters, - advertise, - master_tls.clone(), - )); - } + let masters = master_addresses(&args.masters)?; + // The filers are dialed as the masters are: grpc.client's credentials + // and server allowlist, through the engine's own connector. + let filer_dialer: Option = security + .grpc_client_tls("grpc.client", "grpc.filer")? + .map(|tls| { + Arc::new(move |addr: &str| { + let channel = tonic::transport::Endpoint::new(format!("http://{addr}"))? + .connect_timeout(Duration::from_secs(10)) + .connect_with_connector_lazy(tls::connector(tls.clone())); + Ok(sw_client::FilerClient::from_channel(addr, channel)) + }) as FilerDialer + }); + let conn = Connection { + filers: discover_filers(&masters, &args.filer_group, master_tls.as_ref()).await?, + read_jwt_key: security.jwt_read_key(), + sse: security.sse(), + grpc_tls: None, + https_tls: security.https_client_tls()?, + filer_dialer, + }; + let reads = DirectReads::connect(&conn).await.context("direct reads")?; + tracing::info!( + "reading tables through {reads:?}; filer gRPC {}, volume servers {}; SSE-S3 objects {}", + if conn.filer_dialer.is_some() { + "over TLS" + } else { + "plaintext" + }, + if conn.https_tls.is_some() { + "over HTTPS" + } else { + "over HTTP" + }, + if reads.sse_s3_configured() { + "are readable" + } else { + "need s3.sse.kek or s3.sse.key" + }, + ); + tokio::spawn(follow_filers( + reads.clone(), + masters.clone(), + args.filer_group.clone(), + master_tls.clone(), + )); let fallback: FallbackOptions = args.storage_options.into_iter().collect(); - let mut service = - VectorServer::new(fallback).with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get)); + let mut service = VectorServer::new(fallback) + .with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get)) + .with_direct_reads(reads); if args.read_cache_mb > 0 { service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20)); } @@ -172,6 +278,14 @@ async fn main() -> Result<()> { }, if master_tls.is_some() { "over TLS" } else { "in plaintext" }, ); + // Registered only now, with storage connected and the listener bound, + // so the gateway never finds an engine that cannot serve yet. + let advertise = advertised_addr(&args.listen, &masters); + tokio::spawn(master_client::keep_connected( + masters.clone(), + advertise, + master_tls.clone(), + )); let router = Server::builder() // Match the gateway's 1 GiB cap — put batches over the tonic 4 MiB // default otherwise get rejected before they reach the handler. @@ -200,8 +314,11 @@ mod tests { #[test] fn nprobes_must_be_positive() { - assert!(Args::try_parse_from(["weed-vector", "--nprobes", "0"]).is_err()); - let args = Args::try_parse_from(["weed-vector", "--nprobes", "12"]).unwrap(); + let parse = |nprobes: &str| { + Args::try_parse_from(["weed-vector", "--masters", "m:9333", "--nprobes", nprobes]) + }; + assert!(parse("0").is_err()); + let args = parse("12").unwrap(); assert_eq!(args.nprobes.map(std::num::NonZeroUsize::get), Some(12)); } @@ -210,6 +327,8 @@ mod tests { let args = Args::try_parse_from(normalize_args( [ "weed-vector", + "-master", + "m:9333", "--namespace", "http://localhost:9101", "-filer-signing-key", @@ -221,4 +340,15 @@ mod tests { .unwrap(); assert!(args.namespace.is_some() && args.filer_signing_key.is_some()); } + + #[test] + fn the_masters_are_required() { + assert!(Args::try_parse_from(["weed-vector"]).is_err()); + assert!(master_addresses("").is_err()); + assert!(master_addresses(" , ").is_err()); + assert_eq!( + master_addresses(" m1:9333,,m2:9333 ").unwrap(), + ["m1:9333", "m2:9333"] + ); + } } diff --git a/seaweed-vector/crates/server/src/master_client.rs b/seaweed-vector/crates/server/src/master_client.rs index 6cee66ef7f..eede8a9eec 100644 --- a/seaweed-vector/crates/server/src/master_client.rs +++ b/seaweed-vector/crates/server/src/master_client.rs @@ -9,7 +9,9 @@ use anyhow::{Context, Result}; use tokio::time::sleep; use tonic::transport::{Channel, Endpoint}; -use crate::master_pb::{KeepConnectedRequest, seaweed_client::SeaweedClient}; +use crate::master_pb::{ + KeepConnectedRequest, ListClusterNodesRequest, seaweed_client::SeaweedClient, +}; use crate::security::GrpcTls; fn registration(client_addr: &str) -> KeepConnectedRequest { @@ -340,3 +342,74 @@ mod tests { assert!(grpc_addr("m1:9333.65536").is_err()); } } + +/// The filers of `filer_group` registered with the masters, as gRPC +/// `host:port`s, in registration order; the masters are all asked, since a +/// follower may know fewer than the leader. An error means no master answered. +pub async fn list_filers( + masters: &[String], + filer_group: &str, + tls: Option<&GrpcTls>, +) -> Result> { + let mut filers = Vec::new(); + let mut answered = false; + let mut last = None; + for master in masters { + match filers_of(master, filer_group, tls).await { + Ok(found) => { + answered = true; + for filer in found { + if !filers.contains(&filer) { + filers.push(filer); + } + } + } + Err(err) => { + tracing::warn!("master {master}: listing filers: {err:#}"); + last = Some(err); + } + } + } + match (answered, last) { + (false, Some(err)) => Err(err), + (false, None) => Err(anyhow::anyhow!("no master given")), + _ => Ok(filers), + } +} + +async fn filers_of(master: &str, filer_group: &str, tls: Option<&GrpcTls>) -> Result> { + let channel = connect(master, tls).await?; + let request = ListClusterNodesRequest { + client_type: "filer".into(), + filer_group: filer_group.into(), + ..Default::default() + }; + let nodes = tokio::time::timeout( + CONNECT_TIMEOUT, + SeaweedClient::new(channel).list_cluster_nodes(request), + ) + .await + .context("master did not answer")?? + .into_inner() + .cluster_nodes; + Ok(nodes + .iter() + .filter_map(|node| match filer_grpc_addr(&node.address) { + Ok(addr) => Some(addr), + Err(err) => { + tracing::warn!("master {master} lists filer {:?}: {err:#}", node.address); + None + } + }) + .collect()) +} + +/// A filer registers as `host:port.grpc`, or `host:port` with its gRPC at +/// port + 10000. +pub fn filer_grpc_addr(address: &str) -> Result { + let addr = grpc_addr(address)?; + Ok(addr + .split_once("://") + .map_or(addr.as_str(), |(_, rest)| rest) + .to_string()) +} diff --git a/seaweed-vector/crates/server/src/security.rs b/seaweed-vector/crates/server/src/security.rs index 7b3b269956..023f2cdbb7 100644 --- a/seaweed-vector/crates/server/src/security.rs +++ b/seaweed-vector/crates/server/src/security.rs @@ -5,6 +5,8 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; +use sw_client::HttpsTls; +use weed_lance_worker::sse::SseConfig; const FILE_NAME: &str = "security.toml"; @@ -191,8 +193,11 @@ impl Security { self.raw(key).filter(|v| !v.is_empty()) } + /// A boolean as weed reads one: viper accepts what Go's strconv.ParseBool + /// does, `1`, `t`, `T`, `TRUE`, `true` and `True`. pub fn get_bool(&self, key: &str) -> bool { - self.raw(key).is_some_and(|v| v.trim() == "true") + self.raw(key) + .is_some_and(|v| matches!(v.trim(), "1" | "t" | "T" | "TRUE" | "true" | "True")) } /// The file a key names, read whole. @@ -287,6 +292,47 @@ impl Security { } } +impl Security { + /// `https.client` for volume servers; `None` unless enabled. + pub fn https_client_tls(&self) -> Result> { + if !self.get_bool("https.client.enabled") { + return Ok(None); + } + let (cert, key) = match (self.get("https.client.cert"), self.get("https.client.key")) { + (Some(cert), Some(key)) => ( + Some(self.read("https.client.cert", &cert)?), + Some(self.read("https.client.key", &key)?), + ), + (None, None) => (None, None), + _ => bail!("https.client.cert and https.client.key must both be set"), + }; + Ok(Some(HttpsTls { + ca_pem: match self.get("https.client.ca") { + Some(ca) => Some(self.read("https.client.ca", &ca)?), + None => None, + }, + cert_pem: cert, + key_pem: key, + insecure_skip_verify: self.get_bool("https.client.insecure_skip_verify"), + })) + } + + /// `jwt.signing.read.key`, which volume reads must be signed with. + pub fn jwt_read_key(&self) -> Option { + self.get("jwt.signing.read.key") + } + + /// The S3 gateway's SSE-S3 settings. + pub fn sse(&self) -> SseConfig { + SseConfig { + kek_hex: self.get("s3.sse.kek"), + key: self.get("s3.sse.key"), + passphrase: self.get("s3.sse.kek.passphrase"), + require_commitment: self.get_bool("s3.require_key_commitment"), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index 718aa1f77d..ed472052b7 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use tonic::{Request, Response, Status}; use weed_lance_worker::catalog::TableDescription; use weed_lance_worker::dataset::{self, FallbackOptions}; +use weed_lance_worker::direct_read::{DirectReads, Wrappers}; use weed_lance_worker::jobs::vector_index::is_vector_table; use weed_vector_core::index; use weed_vector_core::query::{self, Query}; @@ -455,6 +456,7 @@ fn resolved(table: &pb::Table, schemes: &[&str]) -> Option { pub struct VectorServer { fallback: FallbackOptions, + direct_reads: Option, read_cache: Option, nprobes: Option, /// URI schemes a table's location may have. @@ -465,6 +467,7 @@ impl VectorServer { pub fn new(fallback: FallbackOptions) -> Self { Self { fallback, + direct_reads: None, read_cache: None, nprobes: None, schemes: &["s3"], @@ -484,6 +487,12 @@ impl VectorServer { self } + /// Read tables from the filers and volume servers instead of the gateway. + pub fn with_direct_reads(mut self, reads: DirectReads) -> Self { + self.direct_reads = Some(reads); + self + } + /// Serve repeated reads of write-once table files from `cache`. pub fn with_read_cache(mut self, cache: ReadCache) -> Self { self.read_cache = Some(cache); @@ -526,17 +535,25 @@ impl VectorServer { async fn open(&self, table: &pb::Table) -> Result { let description = self.describe(table)?; - // The options name the store the bytes come from; the cache is scoped - // by it so the same bucket and path on another endpoint never share. - let cache = self.read_cache.as_ref().map(|cache| { + // Direct reads innermost; the cache sits outside them, so a hit never + // reaches storage. The options name the store the bytes come from + // when reads go through the gateway, so the cache is scoped by it and + // the same bucket and path on another endpoint never share. + let mut wrappers: Vec> = Vec::new(); + if let Some(reads) = &self.direct_reads { + wrappers.push(Arc::new(reads.clone())); + } + if let Some(cache) = &self.read_cache { let options = dataset::table_options(&description, &self.fallback); let endpoint = options .get(ENDPOINT_OPTION) .map(String::as_str) .unwrap_or(""); - Arc::new(cache.scoped(endpoint)) as Arc - }); - let mut dataset = dataset::open_at(&description, &self.fallback, cache) + wrappers.push(Arc::new(cache.scoped(endpoint))); + } + let wrapper = (!wrappers.is_empty()) + .then(|| Arc::new(Wrappers(wrappers)) as Arc); + let mut dataset = dataset::open_at(&description, &self.fallback, wrapper) .await .map_err(open_status)?; if table.version != 0 { diff --git a/seaweed-worker/Cargo.lock b/seaweed-worker/Cargo.lock index f4c08a3c79..f87272e123 100644 --- a/seaweed-worker/Cargo.lock +++ b/seaweed-worker/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -619,11 +654,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.5.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] @@ -896,6 +931,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" @@ -918,6 +959,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -1112,6 +1159,16 @@ dependencies = [ "phf", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.6" @@ -1219,6 +1276,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -1362,6 +1425,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -1369,6 +1444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1412,6 +1488,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1421,6 +1506,33 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling" version = "0.23.0" @@ -1789,7 +1901,7 @@ dependencies = [ "num-traits", "rand 0.9.5", "regex", - "sha2", + "sha2 0.11.0", "uuid", ] @@ -2107,6 +2219,17 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -2123,6 +2246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -2134,7 +2258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] @@ -2192,12 +2316,71 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2261,6 +2444,22 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.11" @@ -2462,6 +2661,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2503,6 +2703,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "glob" version = "0.3.4" @@ -2521,6 +2731,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.4.16" @@ -2609,6 +2830,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -2986,6 +3225,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "io-uring" version = "0.7.14" @@ -3169,6 +3417,30 @@ dependencies = [ "zmij", ] +[[package]] +name = "jsonwebtoken" +version = "10.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" +dependencies = [ + "base64 0.22.1", + "ed25519-dalek", + "getrandom 0.2.17", + "hmac 0.12.1", + "js-sys", + "p256", + "p384", + "pem", + "rand 0.8.8", + "rsa", + "serde", + "serde_json", + "sha2 0.10.9", + "signature", + "simple_asn1", + "zeroize", +] + [[package]] name = "lance" version = "10.0.0" @@ -3729,6 +4001,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lexical-core" @@ -4073,6 +4348,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4097,6 +4388,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -4184,6 +4485,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "opendal" version = "0.57.0" @@ -4328,6 +4635,30 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + [[package]] name = "parking" version = "2.2.1" @@ -4375,6 +4706,25 @@ dependencies = [ "stfu8", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4449,12 +4799,45 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -4506,6 +4889,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -4782,13 +5174,24 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -4803,6 +5206,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -4813,6 +5226,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -5010,13 +5432,13 @@ dependencies = [ "bytes", "futures", "hex", - "hmac", + "hmac 0.13.0", "http 1.5.0", "jiff", "log", "percent-encoding", "sha1 0.11.0", - "sha2", + "sha2 0.11.0", "windows-sys 0.61.2", ] @@ -5115,6 +5537,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + [[package]] name = "ring" version = "0.17.14" @@ -5139,6 +5571,26 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -5215,6 +5667,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.15.1" @@ -5369,6 +5830,20 @@ dependencies = [ "seaweed-worker-core", ] +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -5545,6 +6020,17 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -5581,6 +6067,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -5603,6 +6099,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.20", + "time", +] + [[package]] name = "siphasher" version = "1.0.3" @@ -5652,6 +6160,22 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlparser" version = "0.62.0" @@ -5734,6 +6258,27 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "sw-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "log", + "prost", + "protoc-bin-vendored", + "reqwest 0.12.28", + "rustls", + "rustls-pemfile", + "thiserror 2.0.20", + "tokio", + "tokio-rustls", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + [[package]] name = "syn" version = "2.0.119" @@ -6268,6 +6813,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -6503,19 +7058,32 @@ dependencies = [ name = "weed-lance-worker" version = "0.1.0" dependencies = [ + "aes", + "aes-gcm", "anyhow", "arrow-array", "arrow-cast", "arrow-schema", "async-trait", + "base64 0.22.1", + "bytes", "chrono", "clap", + "ctr", + "flate2", "futures", + "hex", + "hkdf", + "hmac 0.12.1", + "jsonwebtoken", "lance", "lance-datafusion", "lance-index", "lance-linalg", "lance-table", + "md-5 0.10.6", + "moka", + "object_store", "prometheus", "reqwest 0.12.28", "roaring", @@ -6524,10 +7092,13 @@ dependencies = [ "seaweed-worker-sort", "serde", "serde_json", + "sha2 0.10.9", + "sw-client", "tokio", "tonic", "tracing", "tracing-subscriber", + "zstd", ] [[package]] @@ -6791,6 +7362,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/seaweed-worker/Cargo.toml b/seaweed-worker/Cargo.toml index 17f732e2a8..dc170ec983 100644 --- a/seaweed-worker/Cargo.toml +++ b/seaweed-worker/Cargo.toml @@ -6,7 +6,7 @@ # sorting jobs share, so that two of them cannot drift on what an order means. [workspace] resolver = "2" -members = ["crates/core", "crates/lance", "crates/sort"] +members = ["crates/core", "crates/lance", "crates/sort", "crates/sw-client"] [workspace.package] version = "0.1.0" diff --git a/seaweed-worker/crates/lance/Cargo.toml b/seaweed-worker/crates/lance/Cargo.toml index 951200d445..edc660f2e2 100644 --- a/seaweed-worker/crates/lance/Cargo.toml +++ b/seaweed-worker/crates/lance/Cargo.toml @@ -22,6 +22,7 @@ path = "src/main.rs" seaweed-common = { path = "../../../seaweed-common" } seaweed-worker-core = { path = "../core" } seaweed-worker-sort = { path = "../sort" } +sw-client = { path = "../sw-client" } prometheus.workspace = true # Only the S3 backend: the other object stores lance enables by default are # build time this worker never spends. @@ -47,6 +48,25 @@ serde_json = "1" tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true +# Reads straight from the filer and volume servers: object_store's trait, +# the gateway's object attributes and ETags, and a read JWT per volume read. +object_store = "0.13" +bytes = "1" +moka = { version = "0.12", features = ["sync"] } +jsonwebtoken = { version = "10", features = ["rust_crypto"] } +md-5 = "0.10" +hex = "0.4" +base64 = "0.22" +# The S3 gateway's SSE-S3 envelope: AES-256-GCM wraps keys, AES-256-CTR wraps +# bytes, HMAC-SHA256 commits them; the filer's volume encryption is AES-256-GCM. +aes = "0.8" +ctr = "0.9" +aes-gcm = "0.10" +hmac = "0.12" +sha2 = "0.10" +hkdf = "0.12" +flate2 = "1" +zstd = "0.13" [dev-dependencies] tokio = { workspace = true } diff --git a/seaweed-worker/crates/lance/src/direct_read.rs b/seaweed-worker/crates/lance/src/direct_read.rs new file mode 100644 index 0000000000..eee786869b --- /dev/null +++ b/seaweed-worker/crates/lance/src/direct_read.rs @@ -0,0 +1,1687 @@ +//! Reads of a table straight from the filer and the volume servers, skipping +//! the S3 gateway: object bytes, metadata and listings alike, whatever the +//! entry looks like (inline content, chunk manifests, sparse or overlapping +//! chunks, encryption by the filer or by the gateway, objects tiered to +//! remote storage). Writes, copies and deletes still go to the gateway. + +use std::fmt; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, bail}; +use base64::prelude::{BASE64_STANDARD, Engine}; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use futures::StreamExt; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use lance::io::WrappingObjectStore; +use md5::{Digest, Md5}; +use object_store::path::Path; +use object_store::{ + CopyOptions, Error, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, + MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, + PutResult, RenameOptions, Result, +}; +use sw_client::filer_pb::Entry; +use sw_client::{ + ChunkDecoder, FilerClient, GrpcTls, HttpTransport, HttpsTls, ReadOptions, ResolvedChunks, + VolumeReader, VolumeTransport, +}; + +use crate::sse::{EntryDecoder, KEK_PATH, SseConfig, SseS3Keys}; + +const OBJECT_CACHE_ENTRIES: u64 = 1 << 16; +const READ_JWT_TTL_SECS: u64 = 60; +/// Bound on one filer's answer, so a filer that accepts the connection and +/// then says nothing is passed over like one that refused it. +const FILER_TIMEOUT: Duration = Duration::from_secs(30); +/// The gateway's object-level ETag, set when it differs from the entry's md5. +const ETAG_ATTRIBUTE: &str = "Seaweed-X-Amz-ETag"; +const PREFIX_OBJECT_ATTRIBUTE: &str = "x-seaweedfs-prefix-object"; + +#[derive(Clone)] +pub struct DirectReads { + shared: Arc, + /// Set when the source is the cluster's filers, for [`Self::set_filers`]. + filers: Option>, +} + +/// An object as the filer holds it, ready to read. +pub struct Object { + entry: Entry, + size: u64, + etag: Option, + chunks: ResolvedChunks, + decoder: Arc, +} + +/// Where entries and their bytes come from: the filers and volume servers. +#[async_trait::async_trait] +trait Source: Send + Sync { + async fn lookup(&self, dir: &str, name: &str) -> anyhow::Result>; + /// The entries of `dir` whose names start with `prefix`, in name order; + /// empty for a missing directory. + async fn list(&self, dir: &str, prefix: &str) -> anyhow::Result>; + /// Copies a remote-only entry onto the volume servers. + async fn cache_remote(&self, dir: &str, name: &str) -> anyhow::Result; + async fn resolve( + &self, + entry: &Entry, + decoder: Arc, + ) -> anyhow::Result; + async fn read(&self, object: &Object, offset: u64, len: u64) -> anyhow::Result>; + fn name(&self) -> String; +} + +/// The cluster's filers as the masters list them, used one at a time; an +/// unreachable one is passed over for the next. +struct Filers { + clients: RwLock>, + dial: FilerDialer, + current: AtomicUsize, + volumes: VolumeReader, +} + +/// Connects to a filer by its gRPC `host:port`. +pub type FilerDialer = Arc anyhow::Result + Send + Sync>; + +impl Filers { + /// Follows the masters' list: known filers keep their connections, new + /// ones are connected lazily, gone ones are dropped. + fn set_addresses(&self, addresses: &[String]) -> anyhow::Result<()> { + let known = self.clients.read().unwrap().clone(); + let mut clients = Vec::with_capacity(addresses.len()); + for addr in addresses { + clients.push(match known.iter().find(|c| c.addr() == addr) { + Some(client) => client.clone(), + None => (self.dial)(addr)?, + }); + } + let before: Vec<&str> = known.iter().map(|c| c.addr()).collect(); + if before != addresses.iter().map(String::as_str).collect::>() { + tracing::info!("filers: {}", addresses.join(",")); + } + *self.clients.write().unwrap() = clients; + Ok(()) + } + + async fn with<'a, T, F>(&self, what: &str, op: F) -> anyhow::Result + where + F: Fn(FilerClient) -> BoxFuture<'a, anyhow::Result>, + { + let clients = self.clients.read().unwrap().clone(); + if clients.is_empty() { + bail!("no filer is registered with the masters"); + } + let start = self.current.load(Ordering::Relaxed); + let mut last = None; + for attempt in 0..clients.len() { + let i = (start + attempt) % clients.len(); + match tokio::time::timeout(FILER_TIMEOUT, op(clients[i].clone())).await { + Ok(Ok(value)) => { + self.current.store(i, Ordering::Relaxed); + return Ok(value); + } + Ok(Err(err)) if sw_client::is_unavailable(&err) => { + tracing::warn!( + "filer {} unavailable for {what}: {err:#}", + clients[i].addr() + ); + last = Some(err); + } + Ok(Err(err)) => return Err(err), + Err(_) => { + tracing::warn!( + "filer {} did not answer {what} within {FILER_TIMEOUT:?}", + clients[i].addr() + ); + last = Some(anyhow::anyhow!( + "filer {} did not answer {what} within {FILER_TIMEOUT:?}", + clients[i].addr() + )); + } + } + } + Err(last.expect("at least one filer")) + } +} + +#[async_trait::async_trait] +impl Source for Filers { + async fn lookup(&self, dir: &str, name: &str) -> anyhow::Result> { + self.with("lookup", |mut c| { + Box::pin(async move { c.lookup_entry(dir, name).await }) + }) + .await + } + + async fn list(&self, dir: &str, prefix: &str) -> anyhow::Result> { + // One filer attempt per page, so the deadline bounds a page, not the + // whole listing of a large directory. + let mut entries = Vec::new(); + let (mut cursor, mut snapshot) = (String::new(), 0); + loop { + let page = self + .with("list", |mut c| { + let cursor = cursor.clone(); + Box::pin(async move { + match c + .list_entries_page_prefixed(dir, prefix, &cursor, snapshot) + .await + { + Ok(page) => Ok(Some(page)), + Err(err) if is_not_found(&err) => Ok(None), + Err(err) => Err(err), + } + }) + }) + .await?; + let Some(page) = page else { + return Ok(entries); + }; + entries.extend(page.entries); + if page.eof { + return Ok(entries); + } + cursor = page.cursor; + snapshot = page.snapshot_ts_ns; + } + } + + async fn cache_remote(&self, dir: &str, name: &str) -> anyhow::Result { + self.with("cache remote", |mut c| { + Box::pin(async move { c.cache_remote_object(dir, name).await }) + }) + .await + } + + async fn resolve( + &self, + entry: &Entry, + decoder: Arc, + ) -> anyhow::Result { + let options = ReadOptions { + exact: true, + decoder: Some(decoder), + }; + self.with("resolve", |mut c| { + let options = options.clone(); + Box::pin(async move { + sw_client::resolve_chunks(&mut c, &self.volumes, &entry.chunks, &options).await + }) + }) + .await + } + + async fn read(&self, object: &Object, offset: u64, len: u64) -> anyhow::Result> { + let options = ReadOptions { + exact: true, + decoder: Some(object.decoder.clone()), + }; + let read = || { + self.with("read", |mut c| { + let options = options.clone(); + Box::pin(async move { + sw_client::read_resolved( + &mut c, + &self.volumes, + &object.chunks, + &object.entry, + offset, + len, + &options, + ) + .await + }) + }) + }; + match read().await { + Ok(bytes) => Ok(bytes), + // A failed fetch forgets the volume locations it was made + // against, so one more attempt looks them up afresh: a volume + // that moved is found where it is now. + Err(first) => read() + .await + .map_err(|again| again.context(format!("after a first attempt failed: {first:#}"))), + } + } + + fn name(&self) -> String { + self.clients + .read() + .unwrap() + .iter() + .map(|c| c.addr()) + .collect::>() + .join(",") + } +} + +/// The filer answers a listing of a missing directory with its missing-entry +/// error; the filer client's tonic is not this crate's, so it is read as text. +fn is_not_found(err: &anyhow::Error) -> bool { + err.chain() + .any(|cause| cause.to_string().contains("no entry is found")) +} + +fn child(dir: &Path, name: &str) -> Path { + Path::from_iter( + dir.parts() + .chain([object_store::path::PathPart::from(name)]), + ) +} + +#[derive(Default)] +struct Stats { + reads: AtomicU64, + bytes: AtomicU64, + lookups: AtomicU64, + listings: AtomicU64, + failures: AtomicU64, +} + +struct Shared { + source: Arc, + buckets_dir: String, + keys: Arc, + /// Write-once files, resolved once; everything else is looked up per + /// request so reads follow writes. + objects: moka::sync::Cache>, + stats: Stats, +} + +impl fmt::Debug for DirectReads { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DirectReads") + .field("filers", &self.shared.source.name()) + .field("buckets_dir", &self.shared.buckets_dir) + .field("sse_s3", &self.shared.keys) + .finish() + } +} + +/// What direct reads connect to, as the cluster's security.toml configures +/// weed processes. +#[derive(Default)] +pub struct Connection { + /// The filers' gRPC `host:port`s as the masters list them; see + /// [`DirectReads::set_filers`] for later changes. + pub filers: Vec, + /// `jwt.signing.read.key`, signing each volume read. + pub read_jwt_key: Option, + pub sse: SseConfig, + /// `grpc.client`, when the filers' gRPC is under TLS. + pub grpc_tls: Option, + /// `https.client`, when volume servers serve HTTPS. + pub https_tls: Option, + /// Dials the filers in place of `grpc_tls`, for a TLS setup sw-client + /// cannot express, such as a server allowlist. + pub filer_dialer: Option, +} + +impl fmt::Debug for Connection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Connection") + .field("filers", &self.filers) + .field("read_jwt", &self.read_jwt_key.is_some()) + .field("sse", &self.sse) + .field("grpc_tls", &self.grpc_tls) + .field("https_tls", &self.https_tls) + .field("filer_dialer", &self.filer_dialer.is_some()) + .finish() + } +} + +impl DirectReads { + pub async fn connect(conn: &Connection) -> anyhow::Result { + if conn.filers.is_empty() { + bail!("no filer is registered with the masters"); + } + let http = match &conn.https_tls { + Some(tls) => HttpTransport::with_tls(tls)?, + None => HttpTransport::new(), + }; + let transport: Arc = + match conn.read_jwt_key.as_deref().filter(|k| !k.is_empty()) { + Some(key) => Arc::new(JwtTransport { + http, + key: jsonwebtoken::EncodingKey::from_secret(key.as_bytes()), + }), + None => Arc::new(http), + }; + let dial: FilerDialer = match &conn.filer_dialer { + Some(dialer) => dialer.clone(), + None => { + let tls = conn.grpc_tls.clone(); + Arc::new(move |addr| FilerClient::connect_lazy_with(addr, tls.as_ref())) + } + }; + let filers = Arc::new(Filers { + clients: RwLock::new(Vec::new()), + dial, + current: AtomicUsize::new(0), + // Lance caches above this store, so needles are not kept here. + volumes: VolumeReader::with_transport_capped(transport, 0), + }); + filers.set_addresses(&conn.filers)?; + let config = filers + .with("configuration", |mut c| { + Box::pin(async move { c.get_configuration().await }) + }) + .await?; + let kek_file = filers + .lookup(KEK_PATH.0, KEK_PATH.1) + .await + .context("looking up the filer's SSE-S3 KEK")? + .map(|entry| entry.content); + let keys = SseS3Keys::load(&conn.sse, kek_file.as_deref())?; + let mut reads = Self::new(filers.clone(), &config.dir_buckets, Arc::new(keys)); + reads.filers = Some(filers); + reads.report_every(Duration::from_secs(30)); + Ok(reads) + } + + fn new(source: Arc, buckets_dir: &str, keys: Arc) -> Self { + let buckets_dir = match buckets_dir.trim_end_matches('/') { + "" => "/buckets".to_string(), + dir => dir.to_string(), + }; + Self { + shared: Arc::new(Shared { + source, + buckets_dir, + keys, + objects: moka::sync::Cache::new(OBJECT_CACHE_ENTRIES), + stats: Stats::default(), + }), + filers: None, + } + } + + /// The filers as the masters now list them. + pub fn set_filers(&self, addresses: &[String]) -> anyhow::Result<()> { + match &self.filers { + Some(filers) => filers.set_addresses(addresses), + None => Ok(()), + } + } + + /// Whether SSE-S3 objects can be read. + pub fn sse_s3_configured(&self) -> bool { + self.shared.keys.configured() + } + + fn report_every(&self, period: Duration) { + let shared = Arc::downgrade(&self.shared); + tokio::spawn(async move { + let mut last = (0, 0, 0); + loop { + tokio::time::sleep(period).await; + let Some(s) = shared.upgrade() else { return }; + let now = ( + s.stats.reads.load(Ordering::Relaxed), + s.stats.lookups.load(Ordering::Relaxed), + s.stats.failures.load(Ordering::Relaxed), + ); + if now != last { + tracing::info!( + "direct reads: {} reads ({} MiB), {} lookups, {} listings, {} failures", + now.0, + s.stats.bytes.load(Ordering::Relaxed) >> 20, + now.1, + s.stats.listings.load(Ordering::Relaxed), + now.2 + ); + last = now; + } + } + }); + } +} + +impl WrappingObjectStore for DirectReads { + fn wrap(&self, store_prefix: &str, original: Arc) -> Arc { + match store_prefix.strip_prefix("s3$") { + Some(bucket) if !bucket.is_empty() => Arc::new(DirectStore { + inner: original, + bucket: bucket.into(), + reads: self.shared.clone(), + }), + _ => original, + } + } +} + +/// Applies wrappers innermost first. +#[derive(Debug)] +pub struct Wrappers(pub Vec>); + +impl WrappingObjectStore for Wrappers { + fn wrap(&self, store_prefix: &str, original: Arc) -> Arc { + self.0 + .iter() + .fold(original, |store, wrapper| wrapper.wrap(store_prefix, store)) + } +} + +/// Lance names data files `data/.lance`, deletion files +/// `_deletions/--.` and index files +/// `_indices//`; nothing else under a table is write-once. +fn immutable(path: &Path) -> bool { + let parts: Vec<_> = path.parts().collect(); + let Some((name, dirs)) = parts.split_last() else { + return false; + }; + match dirs { + [.., dir] if dir.as_ref() == "data" => name.as_ref().ends_with(".lance"), + [.., dir] if dir.as_ref() == "_deletions" => true, + [.., dir, id] if dir.as_ref() == "_indices" => !id.as_ref().starts_with('_'), + _ => false, + } +} + +fn remote_only(entry: &Entry) -> bool { + entry.chunks.is_empty() + && entry + .remote_entry + .as_ref() + .is_some_and(|remote| remote.remote_size > 0) +} + +/// The entry's size as the gateway computes it, before manifests expand. +fn entry_size(entry: &Entry) -> u64 { + let attributes = entry.attributes.as_ref(); + let mut size = attributes.map_or(0, |a| a.file_size); + if let Some(remote) = &entry.remote_entry + && remote.remote_mtime > attributes.map_or(0, |a| a.mtime) + { + size = size.max(remote.remote_size.max(0) as u64); + } + let chunk_end = entry + .chunks + .iter() + .map(|c| c.offset.max(0) as u64 + c.size) + .max() + .unwrap_or(0); + size.max(chunk_end).max(entry.content.len() as u64) +} + +/// The object's size as the gateway reports it. +fn file_size(entry: &Entry, chunks: &ResolvedChunks) -> u64 { + entry_size(entry).max(chunks.end()) +} + +/// A directory with no object at its bare key. A file that a child write +/// promoted to a directory keeps its data, and a prefix object marks a +/// directory that is also the key without the trailing slash; the gateway +/// serves and lists both as objects. +fn bare_directory(entry: &Entry) -> bool { + entry.is_directory + && entry_size(entry) == 0 + && !entry.extended.contains_key(PREFIX_OBJECT_ATTRIBUTE) +} + +/// The object's ETag as the gateway reports it: its stored S3 ETag, else the +/// remote ETag while the entry holds the remote bytes, else the entry's md5, +/// else derived from the chunks', quoted. +fn etag(entry: &Entry) -> Option { + let quoted = |etag: String| { + if etag.starts_with('"') { + etag + } else { + format!("\"{etag}\"") + } + }; + if let Some(stored) = entry + .extended + .get(ETAG_ATTRIBUTE) + .filter(|v| !v.is_empty()) + .and_then(|v| std::str::from_utf8(v).ok()) + { + return Some(quoted(stored.to_string())); + } + // A synced cache fill, or a remote-only entry; local writes clear the + // sync stamp or add chunks or content. + if let Some(remote) = entry + .remote_entry + .as_ref() + .filter(|r| !r.remote_e_tag.is_empty()) + { + let size = entry.attributes.as_ref().map_or(0, |a| a.file_size); + if size == remote.remote_size.max(0) as u64 + && (remote.last_local_sync_ts_ns > 0 + || (entry.chunks.is_empty() && entry.content.is_empty())) + { + return Some(quoted(remote.remote_e_tag.clone())); + } + } + if let Some(md5) = entry + .attributes + .as_ref() + .map(|a| &a.md5) + .filter(|m| !m.is_empty()) + { + return Some(quoted(hex::encode(md5))); + } + let digests: Vec> = entry + .chunks + .iter() + .map(|c| BASE64_STANDARD.decode(&c.e_tag).unwrap_or_default()) + .collect(); + match digests.as_slice() { + [] => None, + [one] => Some(quoted(hex::encode(one))), + many => { + let joined = Md5::digest(many.concat()); + Some(quoted(format!("{}-{}", hex::encode(joined), many.len()))) + } + } +} + +/// Keeps the whole cause chain in the message object_store shows. +fn generic(err: anyhow::Error) -> Error { + Error::Generic { + store: "DirectReads", + source: format!("{err:#}").into(), + } +} + +fn not_found(location: &Path) -> Error { + Error::NotFound { + path: location.to_string(), + source: "the filer has no such entry".into(), + } +} + +#[derive(Debug)] +struct DirectStore { + inner: Arc, + bucket: Arc, + reads: Arc, +} + +impl fmt::Display for DirectStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "DirectReads({})", self.inner) + } +} + +impl fmt::Debug for Shared { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("DirectReads") + } +} + +impl Object { + fn meta(&self, location: &Path) -> ObjectMeta { + let mtime = self.entry.attributes.as_ref().map_or(0, |a| a.mtime); + ObjectMeta { + location: location.clone(), + last_modified: DateTime::::from_timestamp(mtime, 0).unwrap_or_default(), + size: self.size, + e_tag: self.etag.clone(), + version: None, + } + } +} + +impl DirectStore { + fn filer_path(&self, location: &Path) -> String { + format!("{}/{}/{}", self.reads.buckets_dir, self.bucket, location) + } + + fn directory(&self, prefix: Option<&Path>) -> String { + match prefix { + Some(prefix) if !prefix.as_ref().is_empty() => self.filer_path(prefix), + _ => format!("{}/{}", self.reads.buckets_dir, self.bucket), + } + } + + async fn open(&self, location: &Path) -> Result> { + let path = self.filer_path(location); + let cacheable = immutable(location); + if cacheable && let Some(hit) = self.reads.objects.get(&path) { + return Ok(hit); + } + let object = self.fetch(location, &path).await.map_err(|err| { + // Lance probes for manifests that do not exist. + if !matches!(err, Error::NotFound { .. }) { + self.reads.stats.failures.fetch_add(1, Ordering::Relaxed); + } + err + })?; + if cacheable { + self.reads.objects.insert(path, object.clone()); + } + Ok(object) + } + + async fn fetch(&self, location: &Path, path: &str) -> Result> { + let (dir, name) = path.rsplit_once('/').expect("paths are absolute"); + self.reads.stats.lookups.fetch_add(1, Ordering::Relaxed); + let entry = self + .reads + .source + .lookup(dir, name) + .await + .with_context(|| format!("looking up {path}")) + .map_err(generic)?; + let mut entry = match entry { + Some(entry) if !bare_directory(&entry) => entry, + _ => return Err(not_found(location)), + }; + if remote_only(&entry) { + entry = self + .reads + .source + .cache_remote(dir, name) + .await + .with_context(|| format!("caching remote {path}")) + .map_err(generic)?; + } + let decoder = Arc::new( + EntryDecoder::new(self.reads.keys.clone(), &entry) + .with_context(|| format!("decoding {path}")) + .map_err(generic)?, + ); + let chunks = self + .reads + .source + .resolve(&entry, decoder.clone()) + .await + .with_context(|| format!("resolving {path}")) + .map_err(generic)?; + Ok(Arc::new(Object { + size: file_size(&entry, &chunks), + etag: etag(&entry), + entry, + chunks, + decoder, + })) + } + + async fn read(&self, location: &Path, object: &Object, range: Range) -> Result { + let want = range.end - range.start; + let read = self + .reads + .source + .read(object, range.start, want) + .await + .and_then(|bytes| { + if bytes.len() as u64 == want { + Ok(bytes) + } else { + Err(anyhow::anyhow!("read {} of {want} bytes", bytes.len())) + } + }); + let bytes = read.map_err(|err| { + // The entry may have changed under a cached object. + self.reads.objects.invalidate(&self.filer_path(location)); + self.reads.stats.failures.fetch_add(1, Ordering::Relaxed); + generic(err.context(format!("reading {location} {range:?}"))) + })?; + self.reads.stats.reads.fetch_add(1, Ordering::Relaxed); + self.reads.stats.bytes.fetch_add(want, Ordering::Relaxed); + Ok(bytes.into()) + } + + fn object_meta(&self, prefix: &Path, entry: &Entry) -> ObjectMeta { + ObjectMeta { + location: child(prefix, &entry.name), + last_modified: DateTime::::from_timestamp( + entry.attributes.as_ref().map_or(0, |a| a.mtime), + 0, + ) + .unwrap_or_default(), + size: entry_size(entry), + e_tag: etag(entry), + version: None, + } + } + + async fn list_dir(&self, prefix: Option<&Path>) -> Result> { + self.reads.stats.listings.fetch_add(1, Ordering::Relaxed); + let dir = self.directory(prefix); + self.reads + .source + .list(&dir, "") + .await + .with_context(|| format!("listing {dir}")) + .map_err(generic) + } + + /// Every object under `prefix`, directories first-to-last in name order, + /// after `offset` when given. + async fn list_all( + &self, + prefix: Option, + offset: Option, + ) -> Result> { + let mut objects = Vec::new(); + let mut pending = vec![prefix.unwrap_or_default()]; + while let Some(dir) = pending.pop() { + let entries = self.list_dir(Some(&dir)).await?; + let mut subdirs = Vec::new(); + for entry in entries { + if !bare_directory(&entry) { + let meta = self.object_meta(&dir, &entry); + if offset.as_ref().is_none_or(|o| meta.location > *o) { + objects.push(meta); + } + } + if entry.is_directory { + subdirs.push(child(&dir, &entry.name)); + } + } + pending.extend(subdirs.into_iter().rev()); + } + Ok(objects) + } + + fn stream( + &self, + prefix: Option<&Path>, + offset: Option<&Path>, + ) -> BoxStream<'static, Result> { + let store = DirectStore { + inner: self.inner.clone(), + bucket: self.bucket.clone(), + reads: self.reads.clone(), + }; + let (prefix, offset) = (prefix.cloned(), offset.cloned()); + futures::stream::once(async move { store.list_all(prefix, offset).await }) + .flat_map(|listed| match listed { + Ok(objects) => futures::stream::iter(objects.into_iter().map(Ok)).boxed(), + Err(err) => futures::stream::once(async { Err(err) }).boxed(), + }) + .boxed() + } +} + +#[async_trait::async_trait] +#[deny(clippy::missing_trait_methods)] +impl ObjectStore for DirectStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { + if options.version.is_some() { + return Err(Error::NotSupported { + source: "vector buckets keep no object versions".into(), + }); + } + let object = self.open(location).await?; + let meta = object.meta(location); + options.check_preconditions(&meta)?; + if options.head { + return Ok(GetResult { + payload: GetResultPayload::Stream(futures::stream::empty().boxed()), + meta, + range: 0..0, + attributes: Default::default(), + }); + } + let range = match &options.range { + Some(range) => range + .as_range(object.size) + .map_err(|source| Error::Generic { + store: "DirectReads", + source: source.into(), + })?, + None => 0..object.size, + }; + let bytes = self.read(location, &object, range.clone()).await?; + Ok(GetResult { + payload: GetResultPayload::Stream(futures::stream::once(async { Ok(bytes) }).boxed()), + meta, + range, + attributes: Default::default(), + }) + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> Result> { + let object = self.open(location).await?; + // Nearby ranges share one volume request, as they do through S3. + object_store::coalesce_ranges( + ranges, + |range| { + let object = object.clone(); + async move { + let range = + GetRange::Bounded(range) + .as_range(object.size) + .map_err(|source| Error::Generic { + store: "DirectReads", + source: source.into(), + })?; + self.read(location, &object, range).await + } + }, + object_store::OBJECT_STORE_COALESCE_DEFAULT, + ) + .await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, Result>, + ) -> BoxStream<'static, Result> { + let store = DirectStore { + inner: self.inner.clone(), + bucket: self.bucket.clone(), + reads: self.reads.clone(), + }; + self.inner + .delete_stream(locations) + .inspect(move |deleted| { + if let Ok(location) = deleted { + store.reads.objects.invalidate(&store.filer_path(location)); + } + }) + .boxed() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + self.stream(prefix, None) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, Result> { + self.stream(prefix, Some(offset)) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + let dir = prefix.cloned().unwrap_or_default(); + let mut result = ListResult { + common_prefixes: Vec::new(), + objects: Vec::new(), + }; + for entry in self.list_dir(prefix).await? { + if !bare_directory(&entry) { + result.objects.push(self.object_meta(&dir, &entry)); + } + if entry.is_directory { + result.common_prefixes.push(child(&dir, &entry.name)); + } + } + Ok(result) + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { + self.inner.copy_opts(from, to, options).await + } + + async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> { + self.inner.rename_opts(from, to, options).await + } +} + +/// Signs each volume read with a JWT naming its file id, as the filer does +/// when `jwt.signing.read.key` is set. +struct JwtTransport { + http: HttpTransport, + key: jsonwebtoken::EncodingKey, +} + +#[derive(serde::Serialize)] +struct ReadClaims<'a> { + fid: &'a str, + exp: u64, +} + +impl JwtTransport { + fn token(&self, file_id: &str) -> anyhow::Result { + let exp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() + READ_JWT_TTL_SECS; + Ok(jsonwebtoken::encode( + &jsonwebtoken::Header::default(), + &ReadClaims { fid: file_id, exp }, + &self.key, + )?) + } +} + +#[async_trait::async_trait] +impl VolumeTransport for JwtTransport { + async fn read( + &self, + locations: &[String], + file_id: &str, + range: Option<(u64, u64)>, + ) -> anyhow::Result> { + let token = self.token(file_id)?; + self.http + .read_with_auth(locations, file_id, range, &token) + .await + } + + async fn write( + &self, + _location: &str, + _file_id: &str, + _auth: &str, + _filename: &str, + _data: Bytes, + ) -> anyhow::Result<()> { + bail!("direct reads never write") + } + + fn name(&self) -> &'static str { + "http+jwt" + } +} + +#[cfg(test)] +#[allow(clippy::single_range_in_vec_init)] +mod tests { + use super::*; + use futures::TryStreamExt; + use object_store::ObjectStoreExt; + use object_store::memory::InMemory; + use std::collections::HashMap; + use std::sync::Mutex; + use sw_client::filer_pb::{FileChunk, FuseAttributes, RemoteEntry}; + + #[derive(Default)] + struct Fake { + entries: Mutex>, + bytes: Mutex>>, + lookups: AtomicU64, + cached: AtomicU64, + fail_reads: bool, + } + + #[async_trait::async_trait] + impl Source for Fake { + async fn lookup(&self, dir: &str, name: &str) -> anyhow::Result> { + self.lookups.fetch_add(1, Ordering::Relaxed); + Ok(self + .entries + .lock() + .unwrap() + .get(&format!("{dir}/{name}")) + .cloned()) + } + + async fn list(&self, dir: &str, prefix: &str) -> anyhow::Result> { + let entries = self.entries.lock().unwrap(); + let mut listed: Vec = entries + .iter() + .filter(|(path, _)| { + path.rsplit_once('/') + .is_some_and(|(d, n)| d == dir && n.starts_with(prefix)) + }) + .map(|(_, e)| e.clone()) + .collect(); + listed.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(listed) + } + + async fn cache_remote(&self, dir: &str, name: &str) -> anyhow::Result { + self.cached.fetch_add(1, Ordering::Relaxed); + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(&format!("{dir}/{name}")) + .context("no such entry")?; + let remote = entry.remote_entry.as_mut().unwrap(); + remote.last_local_sync_ts_ns = 1; + let size = remote.remote_size as u64; + entry.chunks = vec![FileChunk { + file_id: "3,0c".into(), + size, + ..Default::default() + }]; + Ok(entry.clone()) + } + + async fn resolve( + &self, + entry: &Entry, + _decoder: Arc, + ) -> anyhow::Result { + ResolvedChunks::from_data_chunks(entry.chunks.clone()) + } + + async fn read(&self, object: &Object, offset: u64, len: u64) -> anyhow::Result> { + if self.fail_reads { + bail!("volume server unreachable"); + } + let bytes = self.bytes.lock().unwrap()[&object.entry.name].clone(); + let end = ((offset + len) as usize).min(bytes.len()); + let mut out = vec![0u8; len as usize]; + out[..end - offset as usize].copy_from_slice(&bytes[offset as usize..end]); + Ok(out) + } + + fn name(&self) -> String { + "fake".into() + } + } + + impl Fake { + fn add(&self, path: &str, bytes: &[u8], edit: impl FnOnce(&mut Entry)) { + let name = path.rsplit_once('/').unwrap().1.to_string(); + let mut entry = Entry { + name: name.clone(), + attributes: Some(FuseAttributes { + file_size: bytes.len() as u64, + mtime: 1_700_000_000, + ..Default::default() + }), + chunks: vec![FileChunk { + file_id: "3,01637037d6".into(), + offset: 0, + size: bytes.len() as u64, + e_tag: BASE64_STANDARD.encode(Md5::digest(bytes)), + ..Default::default() + }], + ..Default::default() + }; + edit(&mut entry); + self.entries.lock().unwrap().insert(path.into(), entry); + self.bytes.lock().unwrap().insert(name, bytes.to_vec()); + } + + fn add_dir(&self, path: &str) { + let name = path.rsplit_once('/').unwrap().1.to_string(); + self.entries.lock().unwrap().insert( + path.into(), + Entry { + name, + is_directory: true, + ..Default::default() + }, + ); + } + } + + const DATA: &str = "vector/idx/data/0101abcd.lance"; + const MANIFEST: &str = "vector/idx/_versions/1.manifest"; + + fn no_keys() -> Arc { + Arc::new(SseS3Keys::load(&SseConfig::default(), None).unwrap()) + } + + async fn store(fake: Arc) -> (Arc, Arc) { + let inner = Arc::new(InMemory::new()); + for path in [DATA, MANIFEST] { + inner + .put(&Path::from(path), Bytes::from_static(b"through S3!").into()) + .await + .unwrap(); + } + let reads = DirectReads::new(fake, "", no_keys()); + (reads.wrap("s3$vb", inner.clone()), inner) + } + + async fn ranges(store: &Arc, path: &str, ranges: &[Range]) -> Vec { + store.get_ranges(&Path::from(path), ranges).await.unwrap() + } + + #[test] + fn only_write_once_files_are_immutable() { + for path in [ + "vector/x/data/0101abcd.lance", + "vector/x/_deletions/3-7-12345.arrow", + "vector/x/_indices/1b6f/index.idx", + ] { + assert!(immutable(&Path::from(path)), "{path}"); + } + for path in [ + "vector/x/_versions/1.manifest", + "vector/x/_latest.manifest", + "vector/x/_refs/tags/weed-vector-query-1.json", + "vector/x/_transactions/0-abc.txn", + "vector/x/data/notes.txt", + "vector/_indices/_versions/1.manifest", + ] { + assert!(!immutable(&Path::from(path)), "{path}"); + } + } + + #[test] + fn s3_keys_map_under_the_buckets_directory() { + let inner: Arc = Arc::new(InMemory::new()); + for (dir, want) in [ + ("", "/buckets/vb/vector/idx/data/a.lance"), + ("/buckets/", "/buckets/vb/vector/idx/data/a.lance"), + ("/srv/buckets", "/srv/buckets/vb/vector/idx/data/a.lance"), + ] { + let reads = DirectReads::new(Arc::new(Fake::default()), dir, no_keys()); + let store = DirectStore { + inner: inner.clone(), + bucket: "vb".into(), + reads: reads.shared.clone(), + }; + assert_eq!( + store.filer_path(&Path::from("vector/idx/data/a.lance")), + want + ); + assert_eq!( + store.directory(None), + want.rsplit_once("/vector").unwrap().0 + ); + assert_eq!( + store.directory(Some(&Path::from("vector/idx"))), + format!("{}/vector/idx", want.rsplit_once("/vector").unwrap().0) + ); + } + let reads = DirectReads::new(Arc::new(Fake::default()), "", no_keys()); + assert!( + reads + .wrap("s3$vb", inner.clone()) + .to_string() + .starts_with("DirectReads(") + ); + for prefix in ["file", "s3$", "gs$vb", "memory"] { + assert!( + !reads + .wrap(prefix, inner.clone()) + .to_string() + .starts_with("DirectReads(") + ); + } + } + + #[tokio::test] + async fn every_object_is_read_from_the_filer() { + let fake = Arc::new(Fake::default()); + fake.add(&format!("/buckets/vb/{DATA}"), b"hello world", |_| {}); + fake.add(&format!("/buckets/vb/{MANIFEST}"), b"filer copy!", |_| {}); + fake.add("/buckets/vb/vector/idx/_latest.manifest", b"latest", |_| {}); + let (store, _) = store(fake.clone()).await; + + assert_eq!( + ranges(&store, DATA, &[0..5, 6..11]).await, + vec![&b"hello"[..], &b"world"[..]] + ); + let whole = store.get(&Path::from(DATA)).await.unwrap(); + assert_eq!(whole.meta.size, 11); + assert_eq!( + whole.meta.e_tag.as_deref(), + Some("\"5eb63bbbe01eeed093cb22bb8f5acdc3\"") + ); + assert_eq!(whole.bytes().await.unwrap(), &b"hello world"[..]); + let tail = store + .get_opts( + &Path::from(DATA), + GetOptions { + range: Some(GetRange::Suffix(5)), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(tail.range, 6..11); + assert_eq!(tail.bytes().await.unwrap(), &b"world"[..]); + assert_eq!( + ranges(&store, MANIFEST, &[0..11]).await, + vec![&b"filer copy!"[..]] + ); + assert_eq!( + store + .get_range(&Path::from("vector/idx/_latest.manifest"), 0..6) + .await + .unwrap(), + &b"latest"[..] + ); + assert_eq!(fake.lookups.load(Ordering::Relaxed), 3); + ranges(&store, DATA, &[0..5]).await; + ranges(&store, MANIFEST, &[0..5]).await; + assert_eq!( + fake.lookups.load(Ordering::Relaxed), + 4, + "write-once files are looked up once, the rest every time" + ); + } + + #[tokio::test] + async fn entries_of_every_shape_are_served() { + let fake = Arc::new(Fake::default()); + fake.add( + "/buckets/vb/vector/idx/data/inline.lance", + b"hello world", + |e| { + e.content = b"hello world".to_vec(); + e.chunks.clear(); + }, + ); + fake.add("/buckets/vb/vector/idx/data/sparse.lance", b"hello", |e| { + e.attributes.as_mut().unwrap().file_size = 8; + }); + fake.add("/buckets/vb/vector/idx/data/sse.lance", b"hello", |e| { + e.extended + .insert("X-Amz-Server-Side-Encryption".into(), b"AES256".to_vec()); + e.extended + .insert("x-seaweedfs-sse-s3-key".into(), b"{}".to_vec()); + }); + fake.add( + "/buckets/vb/vector/idx/data/remote.lance", + b"hello world", + |e| { + e.chunks.clear(); + e.remote_entry = Some(RemoteEntry { + remote_size: 11, + ..Default::default() + }); + }, + ); + fake.add_dir("/buckets/vb/vector/idx/data/dir.lance"); + let (store, _) = store(fake.clone()).await; + + let inline = Path::from("vector/idx/data/inline.lance"); + assert_eq!( + store.get_range(&inline, 6..11).await.unwrap(), + &b"world"[..] + ); + let sparse = Path::from("vector/idx/data/sparse.lance"); + assert_eq!(store.head(&sparse).await.unwrap().size, 8); + assert_eq!( + store.get_range(&sparse, 3..8).await.unwrap(), + &b"lo\0\0\0"[..] + ); + let remote = Path::from("vector/idx/data/remote.lance"); + assert_eq!(store.get_range(&remote, 0..5).await.unwrap(), &b"hello"[..]); + assert_eq!(fake.cached.load(Ordering::Relaxed), 1); + // The decoder is built per object, so an SSE-S3 object without keys + // fails at open, with a reason. + let err = store + .head(&Path::from("vector/idx/data/sse.lance")) + .await + .unwrap_err(); + assert!(err.to_string().contains("SSE-S3"), "{err}"); + assert!(matches!( + store.head(&Path::from("vector/idx/data/dir.lance")).await, + Err(Error::NotFound { .. }) + )); + assert!(matches!( + store + .head(&Path::from("vector/idx/data/missing.lance")) + .await, + Err(Error::NotFound { .. }) + )); + assert!(matches!( + store + .get_opts( + &inline, + GetOptions { + version: Some("1".into()), + ..Default::default() + } + ) + .await, + Err(Error::NotSupported { .. }) + )); + } + + #[tokio::test] + async fn directories_holding_an_object_are_served_and_listed() { + let fake = Arc::new(Fake::default()); + fake.add("/buckets/vb/vector/idx/data/a.lance", b"kept", |e| { + e.is_directory = true; + }); + fake.add("/buckets/vb/vector/idx/data/a.lance/part", b"p", |_| {}); + fake.add("/buckets/vb/vector/idx/data/marker", b"", |e| { + e.is_directory = true; + e.chunks.clear(); + e.extended + .insert(PREFIX_OBJECT_ATTRIBUTE.into(), Vec::new()); + }); + fake.add_dir("/buckets/vb/vector/idx/data/empty"); + let (store, _) = store(fake).await; + + let promoted = Path::from("vector/idx/data/a.lance"); + assert_eq!( + store.get_range(&promoted, 0..4).await.unwrap(), + &b"kept"[..] + ); + assert_eq!( + store + .head(&Path::from("vector/idx/data/marker")) + .await + .unwrap() + .size, + 0 + ); + assert!(matches!( + store.head(&Path::from("vector/idx/data/empty")).await, + Err(Error::NotFound { .. }) + )); + + let data = Path::from("vector/idx/data"); + let all: Vec<_> = store + .list(Some(&data)) + .map_ok(|m| (m.location.to_string(), m.size)) + .try_collect() + .await + .unwrap(); + assert_eq!( + all, + vec![ + ("vector/idx/data/a.lance".to_string(), 4), + ("vector/idx/data/marker".to_string(), 0), + ("vector/idx/data/a.lance/part".to_string(), 1), + ] + ); + let level = store.list_with_delimiter(Some(&data)).await.unwrap(); + assert_eq!( + level + .objects + .iter() + .map(|m| m.location.to_string()) + .collect::>(), + vec!["vector/idx/data/a.lance", "vector/idx/data/marker"] + ); + assert_eq!( + level.common_prefixes, + vec![ + promoted, + Path::from("vector/idx/data/empty"), + Path::from("vector/idx/data/marker"), + ] + ); + } + + #[tokio::test] + async fn metadata_and_preconditions_come_from_the_entry() { + let fake = Arc::new(Fake::default()); + fake.add(&format!("/buckets/vb/{MANIFEST}"), b"filer copy!", |e| { + e.extended.insert(ETAG_ATTRIBUTE.into(), b"abc-2".to_vec()); + }); + fake.add("/buckets/vb/vector/idx/_versions/2.manifest", b"two", |e| { + e.attributes.as_mut().unwrap().md5 = vec![0xab; 16]; + e.chunks[0].e_tag.clear(); + }); + fake.add( + "/buckets/vb/vector/idx/_versions/3.manifest", + b"three", + |e| { + e.chunks = vec![ + FileChunk { + file_id: "3,01".into(), + size: 2, + e_tag: BASE64_STANDARD.encode([1u8; 16]), + ..Default::default() + }, + FileChunk { + file_id: "3,02".into(), + offset: 2, + size: 3, + e_tag: BASE64_STANDARD.encode([2u8; 16]), + ..Default::default() + }, + ]; + }, + ); + fake.add( + "/buckets/vb/vector/idx/_versions/4.manifest", + b"four", + |e| { + e.chunks.clear(); + e.remote_entry = Some(RemoteEntry { + remote_e_tag: "r-4".into(), + remote_size: 4, + ..Default::default() + }); + }, + ); + fake.add( + "/buckets/vb/vector/idx/_versions/5.manifest", + b"five", + |e| { + e.remote_entry = Some(RemoteEntry { + remote_e_tag: "r-5".into(), + remote_size: 4, + ..Default::default() + }); + }, + ); + let (store, _) = store(fake).await; + let path = Path::from(MANIFEST); + let meta = store.head(&path).await.unwrap(); + assert_eq!(meta.e_tag.as_deref(), Some("\"abc-2\"")); + assert_eq!(meta.last_modified.timestamp(), 1_700_000_000); + assert_eq!( + store + .head(&Path::from("vector/idx/_versions/2.manifest")) + .await + .unwrap() + .e_tag + .as_deref(), + Some("\"abababababababababababababababab\"") + ); + let joined = hex::encode(Md5::digest([[1u8; 16], [2u8; 16]].concat())); + assert_eq!( + store + .head(&Path::from("vector/idx/_versions/3.manifest")) + .await + .unwrap() + .e_tag, + Some(format!("\"{joined}-2\"")) + ); + // Cached from remote, the entry still holds the remote bytes; written + // over locally, it no longer does. + assert_eq!( + store + .head(&Path::from("vector/idx/_versions/4.manifest")) + .await + .unwrap() + .e_tag + .as_deref(), + Some("\"r-4\"") + ); + assert_eq!( + store + .head(&Path::from("vector/idx/_versions/5.manifest")) + .await + .unwrap() + .e_tag, + Some(format!("\"{}\"", hex::encode(Md5::digest(b"five")))) + ); + + let get = |options: GetOptions| store.get_opts(&path, options); + assert!( + get(GetOptions { + if_match: Some("\"abc-2\"".into()), + ..Default::default() + }) + .await + .is_ok() + ); + assert!(matches!( + get(GetOptions { + if_match: Some("\"other\"".into()), + ..Default::default() + }) + .await, + Err(Error::Precondition { .. }) + )); + assert!(matches!( + get(GetOptions { + if_none_match: Some("\"abc-2\"".into()), + ..Default::default() + }) + .await, + Err(Error::NotModified { .. }) + )); + let since = DateTime::::from_timestamp(1_700_000_000, 0).unwrap(); + assert!(matches!( + get(GetOptions { + if_modified_since: Some(since), + ..Default::default() + }) + .await, + Err(Error::NotModified { .. }) + )); + assert!( + get(GetOptions { + if_unmodified_since: Some(since), + ..Default::default() + }) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn listings_walk_the_filer_directories() { + let fake = Arc::new(Fake::default()); + fake.add("/buckets/vb/vector/idx/_versions/1.manifest", b"1", |_| {}); + fake.add("/buckets/vb/vector/idx/_versions/2.manifest", b"22", |_| {}); + fake.add("/buckets/vb/vector/idx/_latest.manifest", b"l", |_| {}); + fake.add("/buckets/vb/vector/idx/data/a.lance", b"aaa", |_| {}); + fake.add_dir("/buckets/vb/vector/idx/_versions"); + fake.add_dir("/buckets/vb/vector/idx/data"); + let (store, _) = store(fake).await; + + let versions: Vec<_> = store + .list(Some(&Path::from("vector/idx/_versions"))) + .map_ok(|m| (m.location.to_string(), m.size)) + .try_collect() + .await + .unwrap(); + assert_eq!( + versions, + vec![ + ("vector/idx/_versions/1.manifest".to_string(), 1), + ("vector/idx/_versions/2.manifest".to_string(), 2) + ] + ); + let after: Vec<_> = store + .list_with_offset( + Some(&Path::from("vector/idx")), + &Path::from("vector/idx/_versions/1.manifest"), + ) + .map_ok(|m| m.location.to_string()) + .try_collect() + .await + .unwrap(); + assert_eq!( + after, + vec!["vector/idx/_versions/2.manifest", "vector/idx/data/a.lance"] + ); + + let level = store + .list_with_delimiter(Some(&Path::from("vector/idx"))) + .await + .unwrap(); + assert_eq!( + level.common_prefixes, + vec![ + Path::from("vector/idx/_versions"), + Path::from("vector/idx/data") + ] + ); + assert_eq!( + level + .objects + .iter() + .map(|m| m.location.to_string()) + .collect::>(), + vec!["vector/idx/_latest.manifest"] + ); + let empty: Vec<_> = store + .list(Some(&Path::from("vector/none"))) + .try_collect() + .await + .unwrap(); + assert!(empty.is_empty()); + } + + #[tokio::test] + async fn a_failed_read_is_an_error_and_drops_the_cached_object() { + let fake = Arc::new(Fake { + fail_reads: true, + ..Default::default() + }); + fake.add(&format!("/buckets/vb/{DATA}"), b"hello world", |_| {}); + let (store, _) = store(fake.clone()).await; + for _ in 0..2 { + let err = store + .get_ranges(&Path::from(DATA), &[0..11]) + .await + .unwrap_err(); + assert!( + err.to_string().contains("volume server unreachable"), + "{err}" + ); + } + assert_eq!(fake.lookups.load(Ordering::Relaxed), 2); + } + + #[tokio::test] + async fn deleting_forgets_the_cached_object() { + let fake = Arc::new(Fake::default()); + fake.add(&format!("/buckets/vb/{DATA}"), b"hello world", |_| {}); + let (store, _) = store(fake.clone()).await; + ranges(&store, DATA, &[0..5]).await; + store.delete(&Path::from(DATA)).await.unwrap(); + ranges(&store, DATA, &[0..5]).await; + assert_eq!(fake.lookups.load(Ordering::Relaxed), 2); + } + + #[test] + fn read_jwts_name_the_file_id() { + let transport = JwtTransport { + http: HttpTransport::new(), + key: jsonwebtoken::EncodingKey::from_secret(b"secret"), + }; + let token = transport.token("3,01637037d6").unwrap(); + let mut validation = jsonwebtoken::Validation::new(jsonwebtoken::Algorithm::HS256); + validation.required_spec_claims.clear(); + let claims = jsonwebtoken::decode::( + &token, + &jsonwebtoken::DecodingKey::from_secret(b"secret"), + &validation, + ) + .unwrap() + .claims; + assert_eq!(claims["fid"], "3,01637037d6"); + assert!(claims["exp"].as_u64().unwrap() > 1_700_000_000); + } + + #[tokio::test] + async fn filers_follow_the_masters_and_unreachable_ones_are_passed_over() { + let filers = Filers { + clients: RwLock::new(Vec::new()), + dial: Arc::new(|addr| FilerClient::connect_lazy_with(addr, None)), + current: AtomicUsize::new(0), + volumes: VolumeReader::with_transport_capped(Arc::new(HttpTransport::new()), 0), + }; + let err = filers.lookup("/buckets", "vb").await.unwrap_err(); + assert!(err.to_string().contains("no filer"), "{err:#}"); + + filers + .set_addresses(&["127.0.0.1:1".into(), "127.0.0.1:2".into()]) + .unwrap(); + let err = filers.lookup("/buckets", "vb").await.unwrap_err(); + assert!(sw_client::is_unavailable(&err), "{err:#}"); + assert_eq!(filers.name(), "127.0.0.1:1,127.0.0.1:2"); + + filers + .set_addresses(&["127.0.0.1:2".into(), "127.0.0.1:3".into()]) + .unwrap(); + assert_eq!(filers.name(), "127.0.0.1:2,127.0.0.1:3"); + filers.set_addresses(&[]).unwrap(); + assert!(filers.lookup("/buckets", "vb").await.is_err()); + } +} diff --git a/seaweed-worker/crates/lance/src/lib.rs b/seaweed-worker/crates/lance/src/lib.rs index 0e69a16447..113e548fa7 100644 --- a/seaweed-worker/crates/lance/src/lib.rs +++ b/seaweed-worker/crates/lance/src/lib.rs @@ -7,9 +7,11 @@ pub mod catalog; pub mod dataset; +pub mod direct_read; pub mod jobs; pub mod metrics; pub mod preview; +pub mod sse; pub mod tls; pub mod vector_index; diff --git a/seaweed-worker/crates/lance/src/sse.rs b/seaweed-worker/crates/lance/src/sse.rs new file mode 100644 index 0000000000..d7c48b0b8a --- /dev/null +++ b/seaweed-worker/crates/lance/src/sse.rs @@ -0,0 +1,776 @@ +//! Decodes what the filer and the S3 gateway encrypt, for reads that bypass +//! the gateway. The filer's volume encryption (`cipher_key`) is AES-256-GCM +//! over the chunk, compressed first when `is_compressed`. The gateway's SSE-S3 +//! is AES-256-CTR per chunk under a per-object key, which the chunk's and the +//! object's metadata carry wrapped (AES-256-GCM) by the cluster's KEK; the KEK +//! comes from `s3.sse.kek`, `s3.sse.key` or the filer's `/etc/s3/sse_kek`, as +//! for the gateway. + +use std::collections::HashMap; +use std::io::Read; +use std::sync::{Arc, Mutex}; + +use aes_gcm::aead::Aead; +use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use anyhow::{Context, Result, bail}; +use base64::prelude::{BASE64_STANDARD, Engine}; +use ctr::cipher::{KeyIvInit, StreamCipher, StreamCipherSeek}; +use hkdf::Hkdf; +use hmac::{Hmac, Mac}; +use sha2::Sha256; +use sw_client::ChunkDecoder; +use sw_client::filer_pb::{Entry, FileChunk}; + +type Aes256Ctr = ctr::Ctr128BE; + +const KEY_SIZE: usize = 32; +const IV_SIZE: usize = 16; +const GCM_NONCE_SIZE: usize = 12; +const SSE_S3_KEY_ATTRIBUTE: &str = "x-seaweedfs-sse-s3-key"; +const SSE_HEADER_ATTRIBUTE: &str = "X-Amz-Server-Side-Encryption"; +const AES256: &str = "AES256"; +/// `filer_pb.SSEType`. +const SSE_C: i32 = 1; +const SSE_KMS: i32 = 2; +const SSE_S3: i32 = 3; +pub const KEK_PATH: (&str, &str) = ("/etc/s3", "sse_kek"); +const KEK_WRAPPED_V2_MAGIC: &[u8] = b"SWv2"; +const KEK_SALT_SIZE: usize = 32; +const LEGACY_KEK_SALT: &[u8] = b"seaweedfs-sse-s3-kek-wrapping-v1"; + +/// How the gateway's KEK is configured: `security.toml` `s3.sse.kek`, +/// `s3.sse.key` and `s3.sse.kek.passphrase`, or their `WEED_` environment +/// variables. +#[derive(Clone, Default, Debug)] +pub struct SseConfig { + /// Hex-encoded 256-bit KEK. + pub kek_hex: Option, + /// Any secret; the KEK is derived from it. + pub key: Option, + /// Unwraps a KEK stored encrypted on the filer. + pub passphrase: Option, + /// Refuse objects whose metadata carries no key commitment. + pub require_commitment: bool, +} + +/// The KEK that unwraps SSE-S3 object keys. +pub struct SseS3Keys { + kek: Option<[u8; KEY_SIZE]>, + require_commitment: bool, +} + +impl std::fmt::Debug for SseS3Keys { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SseS3Keys") + .field("configured", &self.kek.is_some()) + .finish() + } +} + +impl SseS3Keys { + /// Resolves the KEK the gateway's way: `kek_hex` (which must match the + /// filer's file when both exist), else `key` (refused while the filer's + /// file exists), else the filer's file, else none. `filer_file` is the + /// content of `/etc/s3/sse_kek` when it exists. + pub fn load(config: &SseConfig, filer_file: Option<&[u8]>) -> Result { + let filer_kek = filer_file + .map(|content| kek_from_filer(content, config.passphrase.as_deref())) + .transpose()?; + let kek = match (&config.kek_hex, &config.key) { + (Some(_), Some(_)) => bail!("only one of s3.sse.kek and s3.sse.key may be set"), + (Some(hex), None) => { + let kek = key_from_hex(hex).context("s3.sse.kek")?; + if filer_kek.is_some_and(|f| f != kek) { + bail!("s3.sse.kek does not match the filer's /etc/s3/sse_kek"); + } + Some(kek) + } + (None, Some(secret)) => { + if filer_kek.is_some() { + bail!("s3.sse.key cannot be used while /etc/s3/sse_kek exists on the filer"); + } + Some(derive_key(secret.as_bytes(), None, b"seaweedfs-sse-s3-kek")) + } + (None, None) => filer_kek, + }; + Ok(Self { + kek, + require_commitment: config.require_commitment, + }) + } + + pub fn configured(&self) -> bool { + self.kek.is_some() + } + + fn unwrap_dek(&self, wrapped: &[u8], nonce: &[u8]) -> Result<[u8; KEY_SIZE]> { + let Some(kek) = &self.kek else { + bail!( + "SSE-S3 is not configured: set s3.sse.kek or s3.sse.key for the worker as for the S3 gateway" + ); + }; + gcm_open(kek, nonce, wrapped) + .context("unwrapping the object key")? + .try_into() + .ok() + .context("the object key is not 256 bits") + } +} + +fn key_from_hex(hex: &str) -> Result<[u8; KEY_SIZE]> { + hex::decode(hex.trim()) + .context("not hex")? + .try_into() + .ok() + .context("not 32 bytes") +} + +fn derive_key(secret: &[u8], salt: Option<&[u8]>, info: &[u8]) -> [u8; KEY_SIZE] { + let mut key = [0u8; KEY_SIZE]; + Hkdf::::new(salt, secret) + .expand(info, &mut key) + .expect("32 bytes fit in one HKDF block"); + key +} + +/// The filer's `/etc/s3/sse_kek`: hex, or wrapped under the passphrase as +/// `magic || salt || nonce || ciphertext` (v2) or `nonce || ciphertext` with +/// a fixed salt (v1). +fn kek_from_filer(content: &[u8], passphrase: Option<&str>) -> Result<[u8; KEY_SIZE]> { + if content.is_empty() { + bail!("the filer's /etc/s3/sse_kek is empty"); + } + let text = std::str::from_utf8(content).context("/etc/s3/sse_kek is not text")?; + let Some(passphrase) = passphrase else { + return key_from_hex(text).context("/etc/s3/sse_kek"); + }; + let unwrapped = (|| { + let raw = BASE64_STANDARD.decode(text.trim())?; + let (salt, payload) = match raw.strip_prefix(KEK_WRAPPED_V2_MAGIC) { + Some(rest) if rest.len() > KEK_SALT_SIZE => rest.split_at(KEK_SALT_SIZE), + _ => (LEGACY_KEK_SALT, &raw[..]), + }; + let wrapping = derive_key(passphrase.as_bytes(), Some(salt), b"kek-wrapping"); + if payload.len() < GCM_NONCE_SIZE { + bail!("wrapped KEK too short"); + } + let (nonce, sealed) = payload.split_at(GCM_NONCE_SIZE); + gcm_open(&wrapping, nonce, sealed) + })(); + match unwrapped { + Ok(kek) => kek + .try_into() + .ok() + .context("the unwrapped KEK is not 256 bits"), + // A KEK written before the passphrase was set is still plain hex. + Err(err) => key_from_hex(text).map_err(|_| err.context("unwrapping /etc/s3/sse_kek")), + } +} + +fn gcm_open(key: &[u8; KEY_SIZE], nonce: &[u8], sealed: &[u8]) -> Result> { + let nonce: [u8; GCM_NONCE_SIZE] = nonce + .try_into() + .ok() + .with_context(|| format!("nonce is {} bytes, not {GCM_NONCE_SIZE}", nonce.len()))?; + Aes256Gcm::new(key.into()) + .decrypt(&Nonce::from(nonce), sealed) + .map_err(|_| anyhow::anyhow!("AES-GCM authentication failed")) +} + +/// The filer's volume encryption: `nonce || ciphertext` under the chunk's key. +fn decrypt_chunk(cipher_key: &[u8], data: &[u8]) -> Result> { + let key: &[u8; KEY_SIZE] = cipher_key + .try_into() + .ok() + .context("the chunk's cipher key is not 256 bits")?; + if data.len() < GCM_NONCE_SIZE { + bail!("encrypted chunk too short"); + } + let (nonce, sealed) = data.split_at(GCM_NONCE_SIZE); + gcm_open(key, nonce, sealed).context("decrypting the chunk") +} + +/// Undoes the filer's compression by the data's magic bytes; other data is +/// returned as it is, as the filer does. +fn decompress(data: Vec) -> Result> { + match data.as_slice() { + [0x1f, 0x8b, ..] => { + let mut out = Vec::new(); + flate2::read::GzDecoder::new(data.as_slice()) + .read_to_end(&mut out) + .context("gunzip")?; + Ok(out) + } + [0x28, 0xb5, 0x2f, 0xfd, ..] => zstd::decode_all(data.as_slice()).context("unzstd"), + _ => Ok(data), + } +} + +/// An SSE-S3 key as the gateway serializes it: the object key wrapped by the +/// KEK, with the IV and the commitment binding the two. +#[derive(serde::Deserialize)] +struct SerializedKey { + algorithm: Option, + #[serde(rename = "encryptedDEK")] + encrypted_dek: String, + nonce: String, + iv: Option, + #[serde(rename = "keyCommitment")] + key_commitment: Option, +} + +struct ObjectKey { + key: [u8; KEY_SIZE], + iv: Option<[u8; IV_SIZE]>, +} + +impl ObjectKey { + fn parse(keys: &SseS3Keys, data: &[u8]) -> Result { + let meta: SerializedKey = serde_json::from_slice(data).context("SSE-S3 metadata")?; + let b64 = |field: &str| { + BASE64_STANDARD + .decode(field) + .context("SSE-S3 metadata base64") + }; + let key = keys.unwrap_dek(&b64(&meta.encrypted_dek)?, &b64(&meta.nonce)?)?; + let iv: Option<[u8; IV_SIZE]> = match &meta.iv { + Some(iv) => Some( + <[u8; IV_SIZE]>::try_from(b64(iv)?.as_slice()) + .ok() + .context("the SSE-S3 IV is not 16 bytes")?, + ), + None => None, + }; + let algorithm = meta.algorithm.as_deref().unwrap_or(AES256); + match (&meta.key_commitment, iv) { + (Some(commitment), Some(iv)) => { + let mut mac = as Mac>::new_from_slice(&key).expect("any key length"); + mac.update(&iv); + mac.update(algorithm.as_bytes()); + mac.verify_slice(&b64(commitment)?) + .map_err(|_| anyhow::anyhow!("the SSE-S3 key commitment does not match"))?; + } + (None, Some(_)) if keys.require_commitment => { + bail!("the SSE-S3 metadata carries no key commitment") + } + _ => {} + } + Ok(Self { key, iv }) + } + + /// Decrypts `data`, the ciphertext from `position` bytes into the CTR + /// stream that starts at this key's IV. + fn decrypt(&self, position: u64, data: &mut [u8]) -> Result<()> { + let iv = self.iv.context("the SSE-S3 key has no IV")?; + let mut ctr = Aes256Ctr::new(&self.key.into(), &iv.into()); + ctr.try_seek(position) + .map_err(|e| anyhow::anyhow!("CTR position {position}: {e}"))?; + ctr.apply_keystream(data); + Ok(()) + } +} + +fn attribute<'a>(entry: &'a Entry, name: &str) -> Option<&'a [u8]> { + entry + .extended + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_slice()) +} + +/// Decodes one entry's chunks: the filer's encryption first, then SSE-S3 with +/// the chunk's own key, or the object's key for chunks written without one. +pub struct EntryDecoder { + keys: Arc, + object: Option, + chunk_keys: Mutex>>, +} + +impl EntryDecoder { + pub fn new(keys: Arc, entry: &Entry) -> Result { + // As the gateway, an object is SSE-S3 only with both the header and + // the key; a header left over from a decrypting copy means nothing. + let object = match ( + attribute(entry, SSE_HEADER_ATTRIBUTE), + attribute(entry, SSE_S3_KEY_ATTRIBUTE), + ) { + (Some(algorithm), Some(key)) if algorithm == AES256.as_bytes() => { + Some(ObjectKey::parse(&keys, key).context("the object's SSE-S3 key")?) + } + _ => None, + }; + Ok(Self { + keys, + object, + chunk_keys: Mutex::new(HashMap::new()), + }) + } + + /// Whether anything about this entry needs decoding. + pub fn is_plain(&self, entry: &Entry) -> bool { + self.object.is_none() + && entry + .chunks + .iter() + .all(|c| c.cipher_key.is_empty() && c.sse_type == 0) + } + + fn chunk_key(&self, chunk: &FileChunk) -> Result> { + // A chunk out of a manifest carries its id as `fid`, not `file_id`. + let id = sw_client::read::file_id_string(chunk).unwrap_or_default(); + if let Some(key) = self.chunk_keys.lock().unwrap().get(&id) { + return Ok(key.clone()); + } + let key = Arc::new( + ObjectKey::parse(&self.keys, &chunk.sse_metadata) + .with_context(|| format!("chunk {id}'s SSE-S3 key"))?, + ); + self.chunk_keys.lock().unwrap().insert(id, key.clone()); + Ok(key) + } +} + +impl ChunkDecoder for EntryDecoder { + fn needs_whole_chunk(&self, chunk: &FileChunk) -> bool { + !chunk.cipher_key.is_empty() + } + + fn decode(&self, chunk: &FileChunk, offset: u64, data: Vec) -> Result> { + let mut data = if chunk.cipher_key.is_empty() { + data + } else { + let plain = decrypt_chunk(&chunk.cipher_key, &data)?; + if chunk.is_compressed { + decompress(plain)? + } else { + plain + } + }; + // Manifests list chunks; the gateway encrypts only the bytes. + if chunk.is_chunk_manifest { + return Ok(data); + } + match chunk.sse_type { + 0 => { + if let Some(object) = &self.object { + object.decrypt(chunk.offset as u64 + offset, &mut data)?; + } + } + SSE_S3 if chunk.sse_metadata.is_empty() => { + let object = self + .object + .as_ref() + .context("an SSE-S3 chunk without a key in an object without one")?; + object.decrypt(chunk.offset as u64 + offset, &mut data)?; + } + SSE_S3 => self.chunk_key(chunk)?.decrypt(offset, &mut data)?, + SSE_C => bail!("SSE-C objects need the customer's key, which direct reads do not have"), + SSE_KMS => bail!("SSE-KMS objects are not supported by direct reads"), + other => bail!("unknown SSE type {other}"), + } + Ok(data) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sw_client::filer_pb::FuseAttributes; + + #[derive(serde::Deserialize)] + struct FixtureChunk { + offset: i64, + ciphertext: String, + sse_metadata: String, + } + + #[derive(serde::Deserialize)] + struct Fixture { + kek_secret: String, + kek_hex: String, + kek_passphrase: String, + kek_wrapped_v1: String, + kek_wrapped_v2: String, + plaintext: String, + object_metadata: String, + chunks: Vec, + whole_ciphertext: String, + cipher_key: String, + cipher_plain: String, + cipher_gzip: String, + } + + /// Written by the gateway's own code: `TestGenerateDirectReadFixture` in + /// weed/s3api. + fn fixture() -> Fixture { + serde_json::from_str(include_str!("../tests/direct_read_fixture.json")).unwrap() + } + + fn un(hex: &str) -> Vec { + hex::decode(hex).unwrap() + } + + fn keys(config: SseConfig, filer_file: Option<&[u8]>) -> Arc { + Arc::new(SseS3Keys::load(&config, filer_file).unwrap()) + } + + fn from_secret(f: &Fixture) -> Arc { + keys( + SseConfig { + key: Some(f.kek_secret.clone()), + ..Default::default() + }, + None, + ) + } + + fn sse_entry(f: &Fixture, with_object_key: bool) -> Entry { + let mut entry = Entry { + name: "a.lance".into(), + attributes: Some(FuseAttributes { + file_size: f.plaintext.len() as u64 / 2, + ..Default::default() + }), + chunks: f + .chunks + .iter() + .enumerate() + .map(|(i, c)| FileChunk { + file_id: format!("3,{i:02x}"), + offset: c.offset, + size: c.ciphertext.len() as u64 / 2, + sse_type: SSE_S3, + sse_metadata: c.sse_metadata.clone().into_bytes(), + ..Default::default() + }) + .collect(), + ..Default::default() + }; + if with_object_key { + entry + .extended + .insert(SSE_HEADER_ATTRIBUTE.into(), AES256.into()); + entry.extended.insert( + SSE_S3_KEY_ATTRIBUTE.into(), + f.object_metadata.clone().into_bytes(), + ); + } + entry + } + + #[test] + fn sse_s3_chunks_decrypt_from_any_offset() { + let f = fixture(); + let plain = un(&f.plaintext); + let entry = sse_entry(&f, false); + let decoder = EntryDecoder::new(from_secret(&f), &entry).unwrap(); + assert!(!decoder.is_plain(&entry)); + for (i, chunk) in entry.chunks.iter().enumerate() { + let ciphertext = un(&f.chunks[i].ciphertext); + assert!(!decoder.needs_whole_chunk(chunk)); + for offset in [0usize, 1, 7, 16, 17, 100, 2047] { + let want = &plain + [chunk.offset as usize + offset..chunk.offset as usize + chunk.size as usize]; + let got = decoder + .decode(chunk, offset as u64, ciphertext[offset..].to_vec()) + .unwrap(); + assert_eq!(got, want, "chunk {i} from {offset}"); + } + } + } + + /// Chunks expanded from a manifest name themselves by `fid`; keyed by the + /// empty `file_id` they would all share one key and IV. + #[test] + fn chunks_out_of_a_manifest_keep_their_own_keys() { + let f = fixture(); + let plain = un(&f.plaintext); + let mut entry = sse_entry(&f, false); + for (i, chunk) in entry.chunks.iter_mut().enumerate() { + chunk.file_id.clear(); + chunk.fid = Some(sw_client::filer_pb::FileId { + volume_id: 3, + file_key: i as u64 + 1, + cookie: 7, + }); + } + let decoder = EntryDecoder::new(from_secret(&f), &entry).unwrap(); + for (i, chunk) in entry.chunks.iter().enumerate() { + let got = decoder + .decode(chunk, 0, un(&f.chunks[i].ciphertext)) + .unwrap(); + let want = &plain[chunk.offset as usize..chunk.offset as usize + chunk.size as usize]; + assert_eq!(got, want, "chunk {i}"); + } + } + + #[test] + fn chunks_without_their_own_key_use_the_objects() { + let f = fixture(); + let plain = un(&f.plaintext); + let whole = un(&f.whole_ciphertext); + let mut entry = sse_entry(&f, true); + for chunk in &mut entry.chunks { + chunk.sse_type = 0; + chunk.sse_metadata.clear(); + } + let decoder = EntryDecoder::new(from_secret(&f), &entry).unwrap(); + let chunk = &entry.chunks[1]; + let start = chunk.offset as usize; + let got = decoder + .decode(chunk, 13, whole[start + 13..start + 500].to_vec()) + .unwrap(); + assert_eq!(got, &plain[start + 13..start + 500]); + + // A header without a key is not encryption, as for the gateway. + entry.extended.remove(SSE_S3_KEY_ATTRIBUTE); + let decoder = EntryDecoder::new(from_secret(&f), &entry).unwrap(); + assert!(decoder.is_plain(&entry)); + assert_eq!( + decoder.decode(chunk, 0, b"as is".to_vec()).unwrap(), + b"as is" + ); + } + + #[test] + fn the_kek_comes_from_hex_a_secret_or_the_filer() { + let f = fixture(); + let entry = sse_entry(&f, false); + let plain = un(&f.plaintext); + let ciphertext = un(&f.chunks[0].ciphertext); + let v1 = f.kek_wrapped_v1.as_bytes(); + let v2 = f.kek_wrapped_v2.as_bytes(); + let hex_file = f.kek_hex.as_bytes(); + let passphrase = Some(f.kek_passphrase.clone()); + let hex = Some(f.kek_hex.clone()); + let secret = Some(f.kek_secret.clone()); + let cases: Vec<(&str, SseConfig, Option<&[u8]>)> = vec![ + ( + "hex", + SseConfig { + kek_hex: hex.clone(), + ..Default::default() + }, + None, + ), + ( + "hex matching the filer", + SseConfig { + kek_hex: hex.clone(), + ..Default::default() + }, + Some(hex_file), + ), + ( + "secret", + SseConfig { + key: secret.clone(), + ..Default::default() + }, + None, + ), + ("filer hex", SseConfig::default(), Some(hex_file)), + ( + "filer v2", + SseConfig { + passphrase: passphrase.clone(), + ..Default::default() + }, + Some(v2), + ), + ( + "filer v1", + SseConfig { + passphrase: passphrase.clone(), + ..Default::default() + }, + Some(v1), + ), + ( + "filer hex with a passphrase", + SseConfig { + passphrase: passphrase.clone(), + ..Default::default() + }, + Some(hex_file), + ), + ]; + for (case, config, file) in cases { + let decoder = EntryDecoder::new(keys(config, file), &entry).unwrap(); + let got = decoder + .decode(&entry.chunks[0], 0, ciphertext.clone()) + .unwrap(); + assert_eq!(got, &plain[..ciphertext.len()], "{case}"); + } + + let refused: Vec<(&str, SseConfig, Option<&[u8]>)> = vec![ + ( + "both", + SseConfig { + kek_hex: hex.clone(), + key: secret.clone(), + ..Default::default() + }, + None, + ), + ( + "secret beside a filer file", + SseConfig { + key: secret.clone(), + ..Default::default() + }, + Some(hex_file), + ), + ( + "hex differing from the filer", + SseConfig { + kek_hex: Some("00".repeat(32)), + ..Default::default() + }, + Some(hex_file), + ), + ( + "wrong passphrase", + SseConfig { + passphrase: Some("other".into()), + ..Default::default() + }, + Some(v2), + ), + ( + "wrapped without a passphrase", + SseConfig::default(), + Some(v2), + ), + ("empty file", SseConfig::default(), Some(b"")), + ]; + for (case, config, file) in refused { + assert!(SseS3Keys::load(&config, file).is_err(), "{case}"); + } + + let unconfigured = keys(SseConfig::default(), None); + assert!(!unconfigured.configured()); + let err = EntryDecoder::new(unconfigured.clone(), &entry) + .unwrap() + .decode(&entry.chunks[0], 0, ciphertext) + .unwrap_err(); + assert!(format!("{err:#}").contains("not configured"), "{err:#}"); + assert!(EntryDecoder::new(unconfigured, &sse_entry(&f, true)).is_err()); + } + + #[test] + fn a_tampered_iv_fails_the_commitment() { + let f = fixture(); + let mut entry = sse_entry(&f, false); + let meta = std::str::from_utf8(&entry.chunks[1].sse_metadata).unwrap(); + let mut value: serde_json::Value = serde_json::from_str(meta).unwrap(); + value["iv"] = serde_json::Value::String(BASE64_STANDARD.encode(b"fedcba9876543210")); + entry.chunks[1].sse_metadata = value.to_string().into_bytes(); + let decoder = EntryDecoder::new(from_secret(&f), &entry).unwrap(); + let err = decoder + .decode(&entry.chunks[1], 0, un(&f.chunks[1].ciphertext)) + .unwrap_err(); + assert!(format!("{err:#}").contains("commitment"), "{err:#}"); + + let mut without = value.clone(); + without.as_object_mut().unwrap().remove("keyCommitment"); + entry.chunks[1].sse_metadata = without.to_string().into_bytes(); + let strict = keys( + SseConfig { + key: Some(f.kek_secret.clone()), + require_commitment: true, + ..Default::default() + }, + None, + ); + assert!( + EntryDecoder::new(strict, &entry) + .unwrap() + .decode(&entry.chunks[1], 0, un(&f.chunks[1].ciphertext)) + .is_err() + ); + } + + #[test] + fn filer_encrypted_chunks_are_decrypted_then_decompressed() { + let f = fixture(); + let plain = un(&f.plaintext); + let entry = Entry::default(); + let decoder = EntryDecoder::new(keys(SseConfig::default(), None), &entry).unwrap(); + // zstd is an Enterprise codec the fixture's generator cannot produce; + // its sealed form is made here with the fixture's key. + let zstd = zstd::encode_all(plain.as_slice(), 0).unwrap(); + let nonce = [7u8; GCM_NONCE_SIZE]; + let mut cipher_zstd = nonce.to_vec(); + cipher_zstd.extend( + Aes256Gcm::new(un(&f.cipher_key).as_slice().into()) + .encrypt(&Nonce::from(nonce), zstd.as_slice()) + .unwrap(), + ); + for (case, data, compressed) in [ + ("plain", un(&f.cipher_plain), false), + ("gzip", un(&f.cipher_gzip), true), + ("zstd", cipher_zstd, true), + ] { + let chunk = FileChunk { + file_id: "3,01".into(), + size: plain.len() as u64, + cipher_key: un(&f.cipher_key), + is_compressed: compressed, + ..Default::default() + }; + assert!(decoder.needs_whole_chunk(&chunk)); + assert_eq!(decoder.decode(&chunk, 0, data).unwrap(), plain, "{case}"); + } + let wrong_key = FileChunk { + cipher_key: vec![1; 32], + ..Default::default() + }; + assert!(decoder.decode(&wrong_key, 0, un(&f.cipher_plain)).is_err()); + } + + #[test] + fn manifests_in_an_encrypted_object_only_undo_the_filers_encryption() { + let f = fixture(); + let plain = un(&f.plaintext); + let entry = sse_entry(&f, true); + let decoder = EntryDecoder::new(from_secret(&f), &entry).unwrap(); + let manifest = FileChunk { + file_id: "3,0m".into(), + is_chunk_manifest: true, + cipher_key: un(&f.cipher_key), + is_compressed: true, + ..Default::default() + }; + assert_eq!( + decoder.decode(&manifest, 0, un(&f.cipher_gzip)).unwrap(), + plain + ); + let plain_manifest = FileChunk { + is_chunk_manifest: true, + ..Default::default() + }; + assert_eq!( + decoder + .decode(&plain_manifest, 0, b"proto".to_vec()) + .unwrap(), + b"proto" + ); + } + + #[test] + fn customer_and_kms_keys_are_refused() { + let decoder = + EntryDecoder::new(keys(SseConfig::default(), None), &Entry::default()).unwrap(); + for sse_type in [SSE_C, SSE_KMS, 9] { + let chunk = FileChunk { + sse_type, + ..Default::default() + }; + assert!( + decoder.decode(&chunk, 0, vec![0; 16]).is_err(), + "{sse_type}" + ); + } + } +} diff --git a/seaweed-worker/crates/lance/tests/direct_read_fixture.json b/seaweed-worker/crates/lance/tests/direct_read_fixture.json new file mode 100644 index 0000000000..17ce3d62dc --- /dev/null +++ b/seaweed-worker/crates/lance/tests/direct_read_fixture.json @@ -0,0 +1,26 @@ +{ + "chunks": [ + { + "offset": 0, + "ciphertext": "2a0ec302f289637828dab37afdfbad45e49589cf8aacbb52c77565813987ebe842cc61d48e26f986ab033bbb051e3ce3cf21a98c8b45780ab2dedb84875eb71d788c6631f5336245ffd3e6da192795968546189447d9ab9a5bdeaf8dabdf7bdb32a2ab6432722ed9b9be3e95343a7f37dfc83a7f958a10a3302c97e9882271e7dd4796dd3c28a0696690e324c5e6010d28cc7d01f9bad6627e0622e37cdd267f92faafc9f78866e0a33b37fdc7b67ff7de4db58247c933854f1eea1dff683594499773bceb425dbe892a6273901c6fe3d5bc98e6e54e8dbe8397dcfef280cfee55c5bf340c81ccf4d4fd47cd40c2cfefb6b9e47d30909a247b3d56d5150c290b5bec299a45371a77b569a4faf1a5971f2e49fcb82e9dbb28fc9d35d95cbcb865c88ca03bec82d943d517751bb3ef583a98c21acd66b59aa0e924a1fa319cc27456fb804cc1fd592c2b8e815be5ceebcfccdf3e3baa4fe730be985bcef0bada6fa654c56477be177a1b486b36bd90a81c6d88106278559ed7d8777163e8e9b179354826ac07a430e2f1f865c01c9c2610d20ebf5bad986fa5d7fc29fef56b2b4ec2de81cf72e8399ad234c88e2a62bda73b0bc3c94181942f03a2b6a433b5554252b510dd3f81ac69868d163433fa7326e5da4e2b1342a18f0948cad8b361b6fefbc8ac426261b3e9d043eeaed46b4ea41ad62282797d5e505cf3f7351470ceb8bf52568f49910570c7b16f32a37b52efa0f97326b9ad2f30a02c84f567514cb004d4782d31eb387a1c3c1d7f3ff184ad6e87f15ed8d93e7693a78b806b818673c229fc9a8d9ab2967903ceea324646bbd0c99412e3217c2f56a116f2d1b6031674bd3d33d197ad702a4812d4f4c4a17fafc7cdce1d4f7b491d0064999ef49f952a8d998baf77a65210ecec2381d8e13695318ca59b4602598fe0f58a2ee1bbd945dcbc0d571bc478d8401fffc1bd47218ffab2a966b247bdb93100ddd4612b18b82267eb3a8af6f8eeb668cbab3f31086dfc93815ade90d458234e88263535927d6c1400443a81cf72f31beddc0625d959de17f9cd8e9e719f430357e71684f59bbb2454b02533e7d15b8d28ccf54f55398c5dd193d12a14adff66cd9f46b2c3a80519cbc5d89c2abec0e80df70d2df4d9934287ec0640d1889c6f4f9f5b639f669753dbce78b72360440bb7821d9bfaef38bfc68dde2eb14f95637d2343537262bd3a677a2cde0031b286f76669e6d6fb62fca09eecc587d29292f0e09cc45e09b0f29672bf96d29b5620e1df94524c92c9b0b69eb16a8991117d00e977e92ff6c2dfae550e9049a88cb1d730318d8ee9b174c79b16bed6e5e1e2a8d21d179a84f2927fdd0e1feb35bbc796b15577e3586bea6276c42f0d34df34e667ef18c1f50bf5bc1a6db86f85b83b67d9162e0d64be141a36c19d9a13da8930f6faac38a000cf3623fa44ae2fb3ddad6345dabd9fad64248343580b92746794e8b30f91a5acba6f641155d7be863c6db494664895c23eb17988b1f6922181d9aa4ac2d06f07250c0e2e3cd1bd660b2dc3827def39f6113fcf6397f806f4d199f51e5f065d577ec20c009c52866b1b98dd2a4d68ad794978e24728986d1205f21c6ee2e32671646fbb49be468ee8b9bb9ca3e0defb483d54c114cc44f4e41240c8c68c357915c878e902e829329efbaf233f787c1b0cc65f9515b2e9565c892e1414d6e8d7bdb39af1f8e16f571f08936df36278a2afcbb44f852c2e1b6979e7c35abc486a610bbe97d5097a032b75a7996ad2b1e0e4dc102ad3f3be6254e9c0c3c9b5f706be25d742c710370568cbe5e54c346f5d65d49142e87d838f0f86a966d47ceb9bdfe0d893704bf70a6d95f5b221bd622d6145ae1875da4c0d53b7b43bd7b2d5804c243d5ac98b901bd949a1d91d2039e1f8865da69a54e349815fe92013104d8d3608c2cf378c9df81543760f646feba0b2e2e2bfbd869f9e19bc75b733efa15eb67b1cd53c2008bf3d1b6fc084497cba21a3f9741904353bbcab2bf8e1e4e86e1054c87bb0f7dca4b4bc52bf0120be8ec638d734712bab7776283c3a5c8fe621092a5ac4fac0d7a1bb754f7e02e9483bcbdc831335ebc96c36fef8f81333bc9986cf6bbe1c598c8c3e53b0f44968999f8eada87e646265dfcec8d31ae82ab611f390e1f7b1b72afb5576e8e682798714ab9172a0b96f0d739c7f21d25742040bfc129002d230fdc2d9f58c8ec9f8bfcadde8dd3d7259a5f263d94b79040ef9fca48c71991e4f4fbfd8c27be71707f632206e2f57eed446963fce6fbd1ee5591eec393512e46ec1a8a27b2aa3d5324f222d03db18df716b4166ef31f2a1d225a6a99d726919d3ac861ec3314a45988abcb21ac876ea1abb8848dafbaccbc1201f318c5911505c24d9edbae57bf58bb3c83a2ffee30cd35e8847e59c93f75b1e2247f4c12115bca558bd9b04f4c273b1001a301703cf749ed859de903c8c1c02f9e83f75c070287cff6112622f8a8847da1351e53f37728edfb0d85c0625aa8ea9265c97b02b89d1c3b74032e16bad92d3aaa3a37648f68bf1aaff61ec6841fc02d72af86aa7a10a280be213096f3fa73a958896a4c99be4e7c8e60b8c6e592bff01348c179643d29122eb773497c1e775837d2cfc7e1e34937e136f029a2a3ee852bcf37baae626e180e74b700c859b56b54251b48ffff05658488d90491815a16f8c5e8ca561a9344de7b9499771a528b1664663e5f29bf0cd6b909da922bc286b3b26ba621b5fa67ec387f974fb4bb9b9a28d450370dd1ebf419bd6c02b0a790e6b3fc44050c16d3e4007c81061079695095973870c1533f910e075bcd7d9776d0199aac5ceac2bdb6bc48e6a6dc24ecc7fa3480ec87880f628980786319e", + "sse_metadata": "{\"algorithm\":\"AES256\",\"encryptedDEK\":\"jJuAjPFVQNo8MbyNEIISVlLhpnxIBKwmr+zUxVk+LRmlBVb+xM0sabyuoISSsjZD\",\"iv\":\"MDEyMzQ1Njc4OWFiY2RlZg==\",\"keyCommitment\":\"oUtzqchnKPjfujBwQUO1cdzeqvgxoM9i8u7AphUIEyY=\",\"keyId\":\"sse-s3-key-1674606777407842260\",\"nonce\":\"qYpP//qAGOxs1hNq\"}" + }, + { + "offset": 2048, + "ciphertext": "f91999d6f6443d817d68210d4ad178ed83d7ad0ef1d48544fac78ca8eb39beef1864cddab3ae099a6d5f908bb454370d78254c832684667e1f080e122b1b18dae91534428d5721c2fb58986b065375077edbba0977aa07083520c807b2402653e189be2b9c8916685040b4e66949675bbb026f50971522723a265077eea1c54772b189b5b271a7f2dd79cb392dbf70148cf944380a4bbe19fbd88919a92eef3377eed498f3bcf5d7f21037ac2ebcb0a3d91886bc1870d195727e344594439fb3f83982e1466efab1073db45b92120a49e70bc0e7a828a21d788fb2e6050102e90cf29e49dd4b1e78edf22e31c6dc33909e2d9d0a3582385760d417fdeaca8e0d31a51a9e1dc1c4300478210e5f5862da7f47263d8b018bf74f45f6fe326df42f78a1f468da714ddadd197266f4bdc3c3b69e6e060392a58eadbacf6bf93820aa13fadcc7c3122ba9370f536d6814e00bbc0a398f5ded3573b9fa0ed799c84b2c18d1dc098b524883a1d109deffe9a1b30e6d0717b10847db8859a647c89952fac8dda24acd57d1197ceb05c897213803b28b8f89158a9fc6dee50761319c19ad41bcd334ad3cbe2323c3ffc3626afadcc9a60dbd1e6749758ecf87745deaee02469843bbd7a7566d9f7e23f4c6519d1daff250efe83044492a55736e37551c805b8a83c0e7f337b670e3d84d8af22ab859cadeccc4d55cf6a9eed13aa3ac1bca813afbcacdbecdfe9553d14596df69f3da3134e43e9c090f8f7e204c6ee8bf3b1ad669513d548feb6c44bc2621346cedd01523b840e823c682579e2d0562a23f3de0641d7e0ea1a768484fb0a6ba8d850f9c4cf50ab92192bc74e21c70fe8049fd15a60169f47b6558f9e2de3aa26e18ee4d5922aa93159b2e4b2971f7a016cf1c680dbfee5bef0c8883bb78093272a86c117961b8472665568601320b5ebd02269d34eeb9b2f6854703dcba6ee791131d9ce674fef8b973afc0fcc6f25e37892a94353ef6940a71c1317aa10deb533f098e2527006c77522330e56fef913da5ae65c2b7d8c7f3be1307e0d5502edee9bacb42b58283555bc039bdf84a0f52ddba4ca742863786ec70e0973f03eece191e3eaa8c09a26b249c42087665380a1bf2cc2834193d407b26ad70d0ae2e9a43f0b3dea7b6326f70fca9070b3f3aa4be1b6433eb32437bd63b357e1f4148ad79ac47418cf3505343b1369d8e742db1f64701f1a75df58189b9667fdcacb1d261fe055be57b521f7798f570ccdb70938046956058cbb720f8a00a2a73fd61481ba9da6f7df7ddb9d33e2c58b679630163dddfd92729c95e60766966f1ef0a07c7b809c0c0a34ac8bcdfab513885dfe74db74d508d8fd391f6e986dae82af6d794ddc33ede0bb5c99271c8673cf6616691fbd9df298c93e79d210a1a4c2adf7e9c731c959efaab691dc89b704e175f2cfda88c0c2f38ff6352ee380f6a7713760c998eb2a66ccd81d142ae4509340879f4be282a643f9da2098cc95ee6ac661fd066f1a1b502834f5825db5744eeddb0f439659086ee7345e6d71215c16feec198907e48a19b42760b47dc7a1b7dae77724b2f7ca8e93008cddea794669e0b5d7b7e959af321e2f04916c3b12107c596a0856fb62a96ffb9b95c8c7697f79af3be336dfddf0fe90407bbfb35d850698cd180df2d09e5acf1d9f687c996e875d7a3f10aa29d89cbcbc03b7c95951b93ba959a175d1ce9b2edada2b715e503efc0da29e9154fe98c91ecf241f8bd6d5f3f2e4300d70cbf516a52d15d287a3721dd16d57ca29a92b2e09c937a22204ffb3f0ebc66584bfd70b4194b63997a4a90d8ba62c616dc19c794b625cb6ee5b1f890bc92d756c00c9caa105337e832991888b4cb0ee1f445dadae212c7ecf0e43c80aa3da6abdbf9ce101deb510dca8cb691f5c6f94648b9ed19d8feaf9fe26641dfc8be145d2ac0a84bfa5b3a4806b829d4c8fd88796093199cb08e2cec722faad44aef87819d69e999a6e0ff4f8e78dd7cdaeb03641cd5eebb210b2eef12d043db40b50246ad7a441574024acd3ff711acdfd8f85abf2f313c716dbb40c154da6958452fef6540a7c64e71647ce146e8120cfc98bd74a4162006d585ef289c2f4750920d9fe0cea926983553a3956d2c451da99d23f15c355a71f066f88f346e5ef6b83d029b61765ad3290a7b504fe1b8513164410b1ff8b073d4487393c0c7b3c1cbf3369fb718f41685c54f0ec483e5aa1e73ab92d948e543329c59e89ff4f4b255b9688cd597c53beb280fe0c0529e4b147401cc3a6da4d8728927fc71aeedc2aaa107d99ada4f489188d85bcb288e669f52c83fbfdd673627253278d69925b88b54343a4c4fa93aeb4b58cf76597d598df2373c2b96ccfde9c905dd03bc48130da42e877b9b6af8cd1d528a29a91576d3d2c20d1ae65c4f2f7492aea7895ef9908e9909d598f650d4bd3d4bf1dfdfe902806abb9f6661648bace8f119102b66366d261610de4463b2a324f8c7007d192c5fd829bd58fd268344ef05ec6c29db85aee26992e10f05a96c93b243373508e4cd9650182b24b615f803713a183d51191f708e9179b7ca7db8220917594f5547083f208f8f854623ca36963c48fee7c7f1776525871afb62a7488d4582538d028c18eec8bc0e7b01679131a05ad267802a102b9b9b99d96b5b8f53fa0f8cb19d43814a132d574c4907df0fe7076f91bf61dda818a746ac4425e0aecea670797faf821a70c040c9044d766144c021b0b9a8942041e12d6876e6aa4568566f3eaea607a06b5d770073b12a09a90bf6f9cffbbe74dbe2a0b60fb62a76ed1cc7e1a48f0b6a1b4e8879be59eeda03b46947afb3e30b84075b429752a5a4207169f500acb7b48a4125d5a21de431d00a338578198be20b08a59c3d6d3d723d9d266eb5dba5ebf94f450fb7a3869c58029cdd6218d4fd0807f9770eea155b88931b85c5b021fed8e6dd250e740cc4fa06d58d1adaaff1301bd5b390c6a59f0c0eb5cd65247e7c145ebb5e65fe8f7d49677d12d37ddc1199f21f68508db95983ee37d4f46cf2d28f575952038fb2e156134e6a935b7e7652594e97692cdc200164da6e2f339a6a5a06997bd25bdabc60d76f0fd32d09357559119c857522af1011598ad9ab63f8c0ceea527a59be0f0951b3b8a69f37a06078f6df825f94e6ad07b92856b33483eca5e4e356e574c728aa219cbc790cc34eb367e2883e09467fb50a8a590861a14add11416e5c27a51e00e3a2dab7b50d3b60f3b83d9e161932b94311c0d600a8689a8d9728855e569fc737b8eee40b427fd55011e099a63d7770460e9e9c413c1310590cc6f9a23e66c592f9c5fa72aaa74477ffb95cb2374917db0414c0cea2833343402696deeb08555f8e78c917ad53c67d4fa5dccd7dd927cdecf598ff0c096b77b4a61d607e9d3eaf48e06a5935d6d7946d716c60d07d2a4024548322da1a5250e788bb1240bc2bc67e594c893ccb594d1efdcb123de0290edd9f56c013e9201ceaef5fa77cbc2f6ba10456bb7f050dda281a528b4398b92a57d6064bf664280032baba7fdc8f9b5f1d869f11208c68e47108071ef7db3f2ededcd087191a4593bb58ff338f183402c46c2bad8e4a458fd42de859def9554fe7c216314ac6a3d4fb1a798bd14f821ae26a84741c4fbd710d6cccdc54a2769cf0bbcc7e6e8baf8faf2c4911fc7419965afd674698ff19147e7e436d7ba7fd25aa875eb9cb9c26c10b31183788b5b3b5e451aa0ecee00c177ae6d6f824c94993fc3ee16ac7bd960717866863aff4293872ffc798787c10ae418e68929248b976facf52990e68e94a2f6661d18ae444439fa314a396805f26bd62e04957098f4f6d10fd47e0eb64c34833075cb9e6dbf46934e1b51900f9feca34c9c6a7efd428c52f80b38800be4ee2e212c30f9e6e9bc21e0dc6e086e7f8e54bca54580687c4a2c53935bb2bab2837e36098224090be8ee78de8b1331b37344c8521f6d4e9f426735198d3d4941c2d25e1d62d99bb8b801bcc76a7f54eedd139f110b690443cbc24e6e42aab865d0e37bf7327a8df608365fafa16449ab13766e3f1c0c9ea9bcd09b814e458bb0432cab7e5394980dc6decd9cd98788250f25e4250d9c6f6d89fc7644f1fcf59018e55b5fc45dafb003f02cc279a96b8e202", + "sse_metadata": "{\"algorithm\":\"AES256\",\"encryptedDEK\":\"IROFhJs/+elQcq7q+pfDM3GIDd5s0G/jMLHJLwV+nToI0ecet+tmfiFEqETSFZQ8\",\"iv\":\"MDEyMzQ1Njc4OWFiY2Rl5g==\",\"keyCommitment\":\"6POJ4Oy2Pd/hcoQUEtIsd3JCxPZrZPwUsvLzWkXX79A=\",\"keyId\":\"sse-s3-key-1674606777407842260\",\"nonce\":\"tjKML4uA8mje1JIU\"}" + } + ], + "cipher_gzip": "43ac83534bef5b644a22350fb1c6e04fc5965493d40cd0f1d8e8bbefee2ff90260eaf5ca9d8e764df62f6d79539a49393c5a91edce5ee6b21cc96e1bbd9fe518e4e6e4fa47a924216eb6bc5e3422db8d39f78c7bfc784626f36673af5d0d76e25759e0d20d85ad3f96648ba7d20442bc415bbd66f2c18ce2ae8a2da134e23fa6054616cc346391dac2dc454b97f69b596218dc4d09caa13ab439afcdb68de1cd9379ad755e16c8b68cf5125c535b4bebfa07dce945fd94345e932a0cab360f0b7dd8f53807abad456b134f55a0754709f7266f85aeb5cb23e50b0b31367f8056a0d9887a97e0797a4dace2fb8d1f96c82018908aa8fa06480c031cc994ba97e57483ee274f6ecde8dc72fb44794dd7be50d68c920b01e81e72f349b154d27940c25e0a1c6df10d9d3d2ae767c4c6b385d9f8eb5aa8402cb7490723d4fc57e94ae6191b422ed6e1954160d6db465e5b17afabdbb27b120d7102ae7edc48b83b536aba096d93fd1876ce88189fc8780068418e7cb43d9cbf5b9070db98445fe33672892977fd089dc1634ef81b98e27da1228231eadc893583dc7d7f1d09af09f1833b0802fbfc704557035738d9e19a6429557e3e2bfc245e5b1e3565e6cbf6bea890f59977c03bf348dcac05da24341dd786c9d87dea0a6fdfd136f724469f294f3a2a2f3897cd54b54e8e63390288450dc42a71ca562dcd4edba55c22493a02df6bade2ba8c2fc696ede9bef14578a07110e85921d8f00d8252f57d82046d1ccb8d12e4115d7a1fd192812e705849e1cdba540ff0305b9ef1e59d870473270689fd6458b1cde41f9ce178529c5c0d3a0f3d67580ab741630abc9c4e33765c9872c261b4a7585e0f9f45b25792d172606e7299951bac1ecc73cb614a9bc0b6b50da66e89af3bed32550a8fb348186086e6805751a5a88ceb6a116300ca7720a0b70516aeadba40c25f54304739cf7a03598c3b0c2d3e190538cf00e7c8594a1f0a3872657dcc1ea7c086fd1e060da4d6d78272b9b9d613fca64a2071d8f755eb1e4d18741884ee13a6fca503495e62510d7f9f36ed14340011fcd7a626db9c96f106986c67a10e9672b5b91b5c86e0f5106dfb1b27d2e4322c6efc187ff141e1ee4dbbb2a46653169a88b7b360ad74ecb3f8a0ef07ba7cbe28c339c62fd1f18fb5568eef", + "cipher_key": "b34344a5441ecd890cfba0b88d6b80e5ce441e4c08ce3026400db5af4301c0dd", + "cipher_plain": "a362d682485d2683a939bae63362f9e5cf7a0066c40e38369622af5975c09ee572b7b35b6a1a65ba756d7847c69d222e5310c5002a0a70debc33c6668e05a89a87c1286c0af1f45334073672f93222e0a087f30547307dc26973031bad8a85118533cec5a9ce2e128f202ab89262de6de33896aff8afc68f34cb15420b24226a53e1d29431200319b761d0f3bbf4aa5370812184ea65cddd2d16509586ddb76a4fc043c9ddf56e57f50d1b053b6ede45d1caa6b9a739d761599befcfc40bc6074b01aade404ca37e1fdbd6e2b07fb015d3332e901ce9a0f3b4d3ddd95824cf1434be3b35b7ca57131b8e36d6a2afd7bbe623159f49d627ede46b6bc166544f42349e7c07fa5659113b1ede695655fd5dfb8c9e8ecbe5a812c2e8524e67eebc678382a063294947d5e40961ebbfb966594b3a3f5af5af62978bbc5359b5eb99d7db55a785e2440934bbe9b80396fe8f5868172fbe4f4143cfcf11039d11e331d11ac7dc0d3c4f93f4852113dd9917d4ca1b70c7c6b0ede65d754a1ce12fa763aa4c8672e627f30fb1d888509ddcf44cbdc38344fffdd514589cdaaf550d79492fa7d3b55d204ec7807794b271c97c04ac2136b1a095f025f8080321533f5c96cbf27e1b2a1d3016b05615840839b409e0d55cd89e2f08714838a98023c0e3935154225e5e234969de1365af6469a40399a11332ca885e228eab3620c71241526d5b3fcb1a8cce2e081f26b5a1abd698145edd0eec69faacee2ddfa941d739cc4fdce5b6c39e163c031c220a0bb906576b0ff6d86b12736a6d7c86c89e5dd028f9c05e4e64fed3737f082a82707c1275d4e01c0da9baf63281a28fb2032e1ea24f4c89e091ac99bba56efde9cedfd6b56c8db5777220e86892828a15e6aafa03cad9139c34eccd5a4f61e6bc8fed368f7699bb65d2fe0c4a15e13ba6f3f1339b218b13759b36974c8c3bea6b69e8d5d76129995366471773db96c241294bc4e7392e06c05d1441aa8758a52fafe9d779b130627485a8716985a6979e15060cba84adf51cbc3bee8f3356f40481b413b0140187a48829d7c6df6cac11c75f6cb6e6df456b44bf28ea192613486d546cf812ded6a03f66297a94d6183b455feddee795d982a493bef8fdd12ba8a7cafb23e6e00248b84c343e9c9039e137134f31944aa0a3a7783526d020aac747acfc53314f2bc1e35b728f3d1692f7b421e61160a3044d309123143cc6694d2e7007d9172b2a578bc31808975c26fdce368517cb3fd34c1be9c8b832a74eec495823752fcf9de97ffded1fe4f280abb2da658a0d6ed59b3980b1644e47b7af8d14a8a1cd65dc7900f18d358f42141d5444964062f8a1519308efe47255a6968bea410a75c5d92b93b8a9247a6714c33aa27c253601079433dc22e78f4f252f3cc99f4fec9859e84a06d2a7b2d88ee2bfdfcf5d19706ddec9db1171d25e4632da84d4a1f084609f3309a0acb9751d1bbf85fcbf3f576765b9d54ed37246526596b2c692afb7518beef48a79a2a5bf976d1488a2d5834459035b6000c43b518ee802b39af9dd8c2ad499437b547e834950145a40aab50cbe15ec32a59b0ebb8d7442a4a1cc4cd2c2e5f813db3b1b12b67d219d65c72ed395d1a67577280ab72d99c28a20180351bc87db6e18f16359731ac733becdbd0ad8d9759a2f2d8e19f49d0c175c7b6598d7dd47badded855f2c1226de86ac473af25d1b7a36d1f100b522d6ae057d3067ab195fd7f6ceade0393ebc7283182d18711bf41a5065987bdc50696ed9e0d2d89f48f326b53c905e85c0922aff03b8e61cf0ee5d7d62af376ac98ec7322e8f96982fd32a471436ba930cb14c4e02b178121c4d9ea694f4b81d4adb62f44f7e5fd4599c04a6ff4723486fe1c5ad0bd9e000b19d566730af0cfdca7b6826899c18fa2b5c64271bf74e4cc653d23eca86d55b8caacf33fd09a6e0cf9f8ad1ac3956f477b9fd2eff4f63790bd4ed557912ccf78d19748eb2f6dd1dc4b1c828e61a87cf1f9b0ce94030d82e676678b271d8cc44da9783b19a2478f9bd813869d18f44398ef1585b924e69ab6687b953c83d6e03ecb74893ce75e42f0f5c903cdac22b74eb5d01d767710798aaf3120e5973891d1e6734a69a141fd2299739dcd8e3059c6da38fb8fe7eb367bba849d648c24f7aecc7e9e9fdb17eda1bb495d409c6f1cc6323fbbefd4364ddd4bf6a11547d3826a1ab46e10de481ecc9a08b6a28d096527ddb411530ff4148cbe52bff77c29afa5719321d1d3a23463e746748741e24522d0f5d33c5028902318ac7207df6151fa8fa58631e7a99e4a8e21f5fbe9ad48e248ce5ce76c7007a0afa8a6753f0d7c9500f91c58dcf4aa2601571e7400e83e4a4a6767cf5f56caf429e0af846429b7a16df723b57baf09c330893b432de98f86da6b20e73a4123302c99cee383d2bbeb9563613b736c4e420601fc8e839b80d8cde664ffcc460228c1e1e5a838c730882616666f49b414a783dc2c5ae45f98e77ddacf45b3d640f3208204e913cecb0eb9337c0b8577b732ccc8458f0fc34045c5de25495b8253761256eb81b857994249c3bca61b9aa5f8298837a92d64dab6efcb59af9dea295358daadc6a8930e87cf88fb64a70fccc6b7c1ebfdfc19f7a52c00a88916f95fc19d8e5cbe5d6e8d16ad4781cafb69db63251630ce1880d93365d8849f999c3b0aa4474d6678dbf7fddfa906c376355ccc6f5f2109d02a2d5c95b9b9ad4a8ea91267357d72be08a5a7b0cb136d3fddba7eeb2f86d5712613bb8ab91459a9c6c44762821cadbb2a200b83eac6a0bdf65cdf30d75d07b47c14b96bda6152289e4505fc0d1b8b0e3aa2220466cd3153c1e79e426aa9289ef0a7d12820bdf18e3f258a97bb37c0630295ca317931433c3577e4ce84b906bfa368aca52d6452bfd9ecfe4f015aa9e8e301661a738b7b1913aaccb356f5d161fa31a695a49821b2f9dbb0d10acac5299fddfd4ddec40f69af60a29b333462e19298c2f7ac12d3f137184e228e68d0845a33bcf53537e6a1a5b296533ebff6b237ad54e676659fdbde00a1c34a4e7fb7196163b99c03404dc171109da0988662ac0d834d01f2d505392b185bd21fd4b13f87066fad216ffd30b595b3b23308f58f27b288d320ffbb420eec02776ec9e08bb895a6223456c1aca9c6113f50d8eb8ca686401f1c88ede5d8018a1d4260c918ecbb4c7ef0b8ae8fade079fc5be58e400a8549e4f875cfb8a3a6b5c4366df5c46a57b30c2cbbd859a8bc090cde741b54d7997118c7da5e180c1b49d8fd16c3b8795f3ffec8526f4202aa58f84028bb5c9d4876ad7fdbe2cb5ac7e10a1e00b33c9fc4ed54de8eab0974eceb18a1636ddc8f1b811e4c755af94af3237073362eecb97bc6c1ad2489f00caba0ee9aafd8431dce4c9e6fcfe1819d78b4b0da5f0f25df405f96bdee1dac86c955af4619fc41c6239449342e362c9210164a3f264f7c7ef8aaf93e6cd7a8e5ca6500c22cfa96a0a7a94f553f96e7586db51b23a8598d047ba9d06b2ea38f164c6d45bd559b62444dfda094fafc714b7cfc75e60bdcb4fbb59a92c596587b3dbdc3e3668c8aa791092a1d0f034d815dc4005181017f1ffa2bc13af1ae3c2f0c4c55492c781fbf01f7964985b4da0196b9f99e708818361e9e8cfb87cdabb8b06ef41632e7ca5e5cbd02bf0f259b7da4b1b33582470dc179759e2d016d08656e491fa63855b7853054314746563d40e6f31cf70f6acda2d0071eca29293c9a201b5c1fa04a3d78ac36f3db7e40dfe4b49e52e03634be51b4996e09ca0268b4064cea79583680eb795aa34510c92f9b0dc89087e01736d0cfe5cc645d7acca83728f49d9d104f2a226f205b2da18a9c37f7ce5153d8b27c7ff0c8a017c0683d281f9de0548a8323f420b3bf703659510330c0dfcf635a8acea8bc5c5bf48411c25dd784fb63690fb2c0b5bb76dc5adec235d2513fdd730dd981d38b4a30be2f76d2153081c1aec1372a61c2bbecd8fbce6451a8a4cbf98f2235f08dfe654805e0fc2a3e322ba8eb7a04adf99eb83edf1bc62c33df4cb01fb500ba898bcf4567f949f618fb92f90b797c3e19285c80a079b07f1ebf03eb805a83e723385360fc867a1736befd865102f2c464ad4c5aee966816010d455fdfb5c62ab67fcb6b45bf622fee6c36d8f0e27e44f1672ca1e529b9385b61ef8e6c24178be84902ca8256fe04f235218c8ea2e5071aeb148a5cddeb64f5c93f04531136cced14b8188dd40dd87cfd2834db25122897ef05ffb202200f02ee247ece09a576537b4d3054dac94eff3a2a7a86f8ec2ff9f466467fa55be735ea3a88a0da82cd103d4a6191f4b8afc909bb076cf2a51d1eeb684d9769dd1a920345897c1ba9d6a42e58427633477d88107e87b3bef00cebee4add9d121505dae0e34b8c112749f74c0a483df43ea29602cb6b1005111983db3befeb0dc5a485b8c63efc12b371f519d4785e46ad9e162c9222452631cac6b1d882a921a5c22f511d8dcf8d356643e8e55b871a8f36eb299bcb69552273824e23ebd60c687e33900461141eca1584e32be3ee48914b6e3ea92db1812e747e0c280f5dfd2b66b2f8ba89086c6f96d54ad1161124821ec82f51ed7e84e7642e726a915bb00a12ad04eac151c2263804c18837ff1f13c67188230fed861cc76d29193fb039f38f98c1c58e142e186a64b5b4cb7569ed2f63f8f45c4bd9e363e0261524aecbc029f1552cee4298a6c68826f5148ba6f57dd8a225bf16e3a815edafe7af644e57bfc216ac510940fac5992d1dc46dcec06bda433b0ea77f9a3d4c2ae97109f1fc0634ab677198d0c17785cec6e70f0021453172a92d55c94ddee75e5f78fb55a0420d6d8988eab301b4fef51789ed324bc2a640a09688fb6dae76d305fee29e7990f90d686150028cbd65d4a4ac14d96a4b9731566b76ccdd010d7c69369613ba4645f10f45c19ecf571617e789c8462130582a836ae89cf19689e7cde2bd45cf26ddbdbbb046625bdc2e088419dff91b8764abaa6a9afedf1200df04a02da13d8568f27a9ad98dba9e2c24eb6b570039d82adce25c3ddfe6c7f61730a2c9bae915af48455a58fd3ad88cbd18fed764350d934aacfb712812d906539584029334eb44c682af7b316ab138db4ddc680183438ab2948b19cd6865b13abcef64b2fd2852c9b59dcf836f71946bf34313cdb269c592f883a3052fb4a06a9f8f29fddd68c8a582174e25d7fe94d84790099765d99d2e86073145685765b4d0923dccfee63ea56b5229c361e601d8c7a95b8e1a15617d334fb4fad9c1634362a21d352145ab4a92c1af8579768d3c1b4a5bd594da2a5dc28492afb7f4dcfa167f18a2a5a2a41beb3ede1edb2430571aa5bad3b48a7882c85ef63a04183bd04cd86895d52368b3834b5cac729de83859ee8c972779118c114bbd876a9ae4c61795c28884303c6be2d7b2a8e7c501fdd540a87531e2bd4f0537f24956a238863edb71d487a7c57abcb5d3a70451ea235c961afbfccbff3c2a25c2e774c9578b06f4f2b86b806d4c2bf9469f57f4d0f11c752aacac33f0d77876427780d982aad8de55d9b8a76d5b554d6514a4b2fa718c62a97b4e1b4a6390f498f7bf3d67e29d58124e88f313473b079fd90b4bbe01a420ff0f0ed5a1ce9cbcda7862a4ce4717cf3e8a3d91ac5eba145469dbf30b7b17f2570026b19d7a0a09072f5d7357b98e128d4860c22d9404d193026338482acb10eeb6f886454fac18d7c911f617d3282410d4f0f4a9aad05c22a735d22cfc41bd0a643e0deeba5c26119c754670a04d96253527c5b276496642d0cf3cbaaf7819750cdddf78b9d52a0ca025cc02f67b6eb3bfb61243eba4d316e2a856f0d510b10a885e995db514c6ac1137511f09d5b6a243a738ec39fd3a2c2d3298a98b4aa9ea8532bb3574056591b2facfd435016685b1d1b1fcb951a014660aad3b63b837722d5c97946dc31619da7b391cb435f9a96e567bd193b26b146bf6341496a6920ede2de2d3abbfefc9afa8da80ac17f17f880546d6e759d7b3074979386aeee9733f9fa559404f9e014275fbb49f78ba869cb6c7c5dc064e2b581ed272a89acd3193697320740946ed5e5281c70bded9a20b481b5d7543eebbac494f54a84179545024da55d9209628d169c2e475ac2ea5fbbd0f9303e6cbf42f44099b1744aee96ed43aa4d6a52fc779b0438b3cabfd09d770c52adbc72961d8e6e543f1ee3089b991a08354bf101d1c68cf3ed023c4dae063195bbba1c1b2b6e89a27cbf6b1bde7e5d70ebe7ef31dfe2ad4b536e8689790a966e3d560249756d6539f3f54f236362709b2a6704401f5bc0bd112850a0a213430a0043ea0f74e910e075216e04bcd1099fc554f3446b2f8d65bb1e2ab14958ab610c3a7f3aae0b5ba6984884a65cc8791b9074fcc3577ed31bd31ef45b3392a73190daf4fd815438a822b173d67d31652de9918eb83f04c9774c0ceef6cd4e4ccbe5b23fef9c72336741a5a1b23f21b8f15d92b28e160d2dc8f5676315a5ad6d8fd3a5896bd53845a74909b9279b090c3adeb003f319e15a33e6a3d28bd19b7200e2cc8a94613f1eb74d6d8665959d33e73525ddadc70a39fd9e8b2bccc76fa4ea1c82c2267d548eedd2164a97066566de8f409e8aa730d07c94f2a662a2caeddab1bf7aa2a4f1f23efc4cde13697d6edb520a822c49464b7181f48cbdad9776fa8dd98515f7c422a51e9d9bb2053fb8c0fff622d623abae0b74a86b1ff7548412292bdbee1e9830c84f2cae52e1cad5ad9ede2dc0a885de485ee4adadb2840780deba03ebfa85b0dbcaf0d739b868b4fb4dc6d55f5d333351c1bc39feb91c669e4eadcaf25ae3942bcac8542694c0385227b23bb736da5fd1b65b4345cf46fd081c8ef63295b1fee7e77000c1554f6ad771025b0eb3888b5fcda3f142325fb236758975ba14c042ecbc69851228bfdf201be46ff142f5d9ad5a79fdf762f9606b2233d53cff7b4028f8d0eec6487c38b9b57fe13103c7425f4e63736be0b9d5bbdf8e75e8f1a9b41a72bf7691526aaa91f3c7046d8da6f195d9763da79bae278777a1edf1d7e477", + "commitment_sample": "a14b73a9c86728f8dfba30704143b571dcdeaaf831a0cf62f2eec0a615081326", + "kek_hex": "814e33b95dd38da733a0f98aa4045e350107136d568257228ac27e5689c70308", + "kek_passphrase": "pass-phrase", + "kek_secret": "s3vectors-test", + "kek_wrapped_v1": "yH4CEgYyWUYSGNgAnXAQPm7LC4TkmBcTqQGOOoF3B7aKEVyL41Xoya7fsVXGne2cBoYSPRwlVP6x6KZw", + "kek_wrapped_v2": "U1d2MgrNBSvQd7lXUkHwq8CoZw6dBJOH3AAh1HXexumBnCzU3IhF3goZCKjxSmgpeWU10psw5J5tTolPNZoRKdLsJ08wSPMygpGFzH5FFAGmc7cYada6g4TGnvsKWF7y", + "object_metadata": "{\"algorithm\":\"AES256\",\"encryptedDEK\":\"/BQZt87OjM4PBQ1xMgp1BLGzLeF/azzYEIJElIsA5E9Tr5Jb/Y/cwLz/tSfHiJhj\",\"iv\":\"MDEyMzQ1Njc4OWFiY2RlZg==\",\"keyCommitment\":\"oUtzqchnKPjfujBwQUO1cdzeqvgxoM9i8u7AphUIEyY=\",\"keyId\":\"sse-s3-key-1674606777407842260\",\"nonce\":\"2KY1nIo6MR6c1dwy\"}", + "plaintext": "00070e151c232a31383f464d545c636a71787f868d949ba2a9b0b8bfc6cdd4dbe2e9f0f7fe050c141b222930373e454c535a616870777e858c939aa1a8afb6bdc4ccd3dae1e8eff6fd040b121920282f363d444b525960676e757c848b9299a0a7aeb5bcc3cad1d8e0e7eef5fc030a11181f262d343c434a51585f666d747b828990989fa6adb4bbc2c9d0d7dee5ecf4fb020910171e252c333a414850575e656c737a81888f969da4acb3bac1c8cfd6dde4ebf2f900080f161d242b323940474e555c646b727980878e959ca3aab1b8c0c7ced5dce3eaf1f8ff060d141c232a31383f464d545b626970787f868d949ba2a9b0b7bec5ccd4dbe2e9f0f7fe050c131a212830373e454c535a61686f767d848c939aa1a8afb6bdc4cbd2d9e0e8eff6fd040b121920272e353c444b525960676e757c838a9198a0a7aeb5bcc3cad1d8dfe6edf4fc030a11181f262d343b424950585f666d747b828990979ea5acb4bbc2c9d0d7dee5ecf3fa010810171e252c333a41484f565d646c737a81888f969da4abb2b9c0c8cfd6dde4ebf2f900070e151c242b323940474e555c636a717880878e959ca3aab1b8bfc6cdd4dce3eaf1f8ff060d141b222930383f464d545b626970777e858c949ba2a9b0b7bec5ccd3dae1e8f0f7fe050c131a21282f363d444c535a61686f767d848b9299a0a8afb6bdc4cbd2d9e0e7eef5fc040b121920272e353c434a515860676e757c838a91989fa6adb4bcc3cad1d8dfe6edf4fb020910181f262d343b424950575e656c747b828990979ea5acb3bac1c8d0d7dee5ecf3fa01080f161d242c333a41484f565d646b727980888f969da4abb2b9c0c7ced5dce4ebf2f900070e151c232a313840474e555c636a71787f868d949ca3aab1b8bfc6cdd4dbe2e9f0f8ff060d141b222930373e454c545b626970777e858c939aa1a8b0b7bec5ccd3dae1e8eff6fd040c131a21282f363d444b525960686f767d848b9299a0a7aeb5bcc4cbd2d9e0e7eef5fc030a111820272e353c434a51585f666d747c838a91989fa6adb4bbc2c9d0d8dfe6edf4fb020910171e252c343b424950575e656c737a818890979ea5acb3bac1c8cfd6dde4ecf3fa01080f161d242b323940484f565d646b727980878e959ca4abb2b9c0c7ced5dce3eaf1f800070e151c232a31383f464d545c636a71787f868d949ba2a9b0b8bfc6cdd4dbe2e9f0f7fe050c141b222930373e454c535a616870777e858c939aa1a8afb6bdc4ccd3dae1e8eff6fd040b121920282f363d444b525960676e757c848b9299a0a7aeb5bcc3cad1d8e0e7eef5fc030a11181f262d343c434a51585f666d747b828990989fa6adb4bbc2c9d0d7dee5ecf4fb020910171e252c333a414850575e656c737a81888f969da4acb3bac1c8cfd6dde4ebf2f900080f161d242b323940474e555c646b727980878e959ca3aab1b8c0c7ced5dce3eaf1f8ff060d141c232a31383f464d545b626970787f868d949ba2a9b0b7bec5ccd4dbe2e9f0f7fe050c131a212830373e454c535a61686f767d848c939aa1a8afb6bdc4cbd2d9e0e8eff6fd040b121920272e353c444b525960676e757c838a9198a0a7aeb5bcc3cad1d8dfe6edf4fc030a11181f262d343b424950585f666d747b828990979ea5acb4bbc2c9d0d7dee5ecf3fa010810171e252c333a41484f565d646c737a81888f969da4abb2b9c0c8cfd6dde4ebf2f900070e151c242b323940474e555c636a717880878e959ca3aab1b8bfc6cdd4dce3eaf1f8ff060d141b222930383f464d545b626970777e858c949ba2a9b0b7bec5ccd3dae1e8f0f7fe050c131a21282f363d444c535a61686f767d848b9299a0a8afb6bdc4cbd2d9e0e7eef5fc040b121920272e353c434a515860676e757c838a91989fa6adb4bcc3cad1d8dfe6edf4fb020910181f262d343b424950575e656c747b828990979ea5acb3bac1c8d0d7dee5ecf3fa01080f161d242c333a41484f565d646b727980888f969da4abb2b9c0c7ced5dce4ebf2f900070e151c232a313840474e555c636a71787f868d949ca3aab1b8bfc6cdd4dbe2e9f0f8ff060d141b222930373e454c545b626970777e858c939aa1a8b0b7bec5ccd3dae1e8eff6fd040c131a21282f363d444b525960686f767d848b9299a0a7aeb5bcc4cbd2d9e0e7eef5fc030a111820272e353c434a51585f666d747c838a91989fa6adb4bbc2c9d0d8dfe6edf4fb020910171e252c343b424950575e656c737a818890979ea5acb3bac1c8cfd6dde4ecf3fa01080f161d242b323940484f565d646b727980878e959ca4abb2b9c0c7ced5dce3eaf1f800070e151c232a31383f464d545c636a71787f868d949ba2a9b0b8bfc6cdd4dbe2e9f0f7fe050c141b222930373e454c535a616870777e858c939aa1a8afb6bdc4ccd3dae1e8eff6fd040b121920282f363d444b525960676e757c848b9299a0a7aeb5bcc3cad1d8e0e7eef5fc030a11181f262d343c434a51585f666d747b828990989fa6adb4bbc2c9d0d7dee5ecf4fb020910171e252c333a414850575e656c737a81888f969da4acb3bac1c8cfd6dde4ebf2f900080f161d242b323940474e555c646b727980878e959ca3aab1b8c0c7ced5dce3eaf1f8ff060d141c232a31383f464d545b626970787f868d949ba2a9b0b7bec5ccd4dbe2e9f0f7fe050c131a212830373e454c535a61686f767d848c939aa1a8afb6bdc4cbd2d9e0e8eff6fd040b121920272e353c444b525960676e757c838a9198a0a7aeb5bcc3cad1d8dfe6edf4fc030a11181f262d343b424950585f666d747b828990979ea5acb4bbc2c9d0d7dee5ecf3fa010810171e252c333a41484f565d646c737a81888f969da4abb2b9c0c8cfd6dde4ebf2f900070e151c242b323940474e555c636a717880878e959ca3aab1b8bfc6cdd4dce3eaf1f8ff060d141b222930383f464d545b626970777e858c949ba2a9b0b7bec5ccd3dae1e8f0f7fe050c131a21282f363d444c535a61686f767d848b9299a0a8afb6bdc4cbd2d9e0e7eef5fc040b121920272e353c434a515860676e757c838a91989fa6adb4bcc3cad1d8dfe6edf4fb020910181f262d343b424950575e656c747b828990979ea5acb3bac1c8d0d7dee5ecf3fa01080f161d242c333a41484f565d646b727980888f969da4abb2b9c0c7ced5dce4ebf2f900070e151c232a313840474e555c636a71787f868d949ca3aab1b8bfc6cdd4dbe2e9f0f8ff060d141b222930373e454c545b626970777e858c939aa1a8b0b7bec5ccd3dae1e8eff6fd040c131a21282f363d444b525960686f767d848b9299a0a7aeb5bcc4cbd2d9e0e7eef5fc030a111820272e353c434a51585f666d747c838a91989fa6adb4bbc2c9d0d8dfe6edf4fb020910171e252c343b424950575e656c737a818890979ea5acb3bac1c8cfd6dde4ecf3fa01080f161d242b323940484f565d646b727980878e959ca4abb2b9c0c7ced5dce3eaf1f800070e151c232a31383f464d545c636a71787f868d949ba2a9b0b8bfc6cdd4dbe2e9f0f7fe050c141b222930373e454c535a616870777e858c939aa1a8afb6bdc4ccd3dae1e8eff6fd040b121920282f363d444b525960676e757c848b9299a0a7aeb5bcc3cad1d8e0e7eef5fc030a11181f262d343c434a51585f666d747b828990989fa6adb4bbc2c9d0d7dee5ecf4fb020910171e252c333a414850575e656c737a81888f969da4acb3bac1c8cfd6dde4ebf2f900080f161d242b323940474e555c646b727980878e959ca3aab1b8c0c7ced5dce3eaf1f8ff060d141c232a31383f464d545b626970787f868d949ba2a9b0b7bec5ccd4dbe2e9f0f7fe050c131a212830373e454c535a61686f767d848c939aa1a8afb6bdc4cbd2d9e0e8eff6fd040b121920272e353c444b525960676e757c838a9198a0a7aeb5bcc3cad1d8dfe6edf4fc030a11181f262d343b424950585f666d747b828990979ea5acb4bbc2c9d0d7dee5ecf3fa010810171e252c333a41484f565d646c737a81888f969da4abb2b9c0c8cfd6dde4ebf2f900070e151c242b323940474e555c636a717880878e959ca3aab1b8bfc6cdd4dce3eaf1f8ff060d141b222930383f464d545b626970777e858c949ba2a9b0b7bec5ccd3dae1e8f0f7fe050c131a21282f363d444c535a61686f767d848b9299a0a8afb6bdc4cbd2d9e0e7eef5fc040b121920272e353c434a515860676e757c838a91989fa6adb4bcc3cad1d8dfe6edf4fb020910181f262d343b424950575e656c747b828990979ea5acb3bac1c8d0d7dee5ecf3fa01080f161d242c333a41484f565d646b727980888f969da4abb2b9c0c7ced5dce4ebf2f900070e151c232a313840474e555c636a71787f868d949ca3aab1b8bfc6cdd4dbe2e9f0f8ff060d141b222930373e454c545b626970777e858c939aa1a8b0b7bec5ccd3dae1e8eff6fd040c131a21282f363d444b525960686f767d848b9299a0a7aeb5bcc4cbd2d9e0e7eef5fc030a111820272e353c434a51585f666d747c838a91989fa6adb4bbc2c9d0d8dfe6edf4fb020910171e252c343b424950575e656c737a818890979ea5acb3bac1c8cfd6dde4ecf3fa01080f161d242b323940484f565d646b727980878e959ca4abb2b9c0c7ced5dce3eaf1f800070e151c232a31383f464d545c636a71787f868d949ba2a9b0b8bfc6cdd4dbe2e9f0f7fe050c141b222930373e454c535a616870777e858c939aa1a8afb6bdc4ccd3dae1e8eff6fd040b121920282f363d444b525960676e757c848b9299a0a7aeb5bcc3cad1d8e0e7eef5fc030a11181f262d343c434a51585f666d747b828990989fa6adb4bbc2c9d0d7dee5ecf4fb020910171e252c333a414850575e656c737a81888f969da4acb3bac1c8cfd6dde4ebf2f900080f161d242b323940474e555c646b727980878e959ca3aab1b8c0c7ced5dce3eaf1f8ff060d141c232a31383f464d545b626970787f868d949ba2a9b0b7bec5ccd4dbe2e9f0f7fe050c131a212830373e454c535a61686f767d848c939aa1a8afb6bdc4cbd2d9e0e8eff6fd040b121920272e353c444b525960676e757c838a9198a0a7aeb5bcc3cad1d8dfe6edf4fc030a11181f262d343b424950585f666d747b828990979ea5acb4bbc2c9d0d7dee5ecf3fa010810171e252c333a41484f565d646c737a81888f969da4abb2b9c0c8cfd6dde4ebf2f900070e151c242b323940474e555c636a717880878e959ca3aab1b8bfc6cdd4dce3eaf1f8ff060d141b222930383f464d545b626970777e858c949ba2a9b0b7bec5ccd3dae1e8f0f7fe050c131a21282f363d444c535a61686f767d848b9299a0a8afb6bdc4cbd2d9e0e7eef5fc040b121920272e353c434a515860676e757c838a91989fa6adb4bcc3cad1d8dfe6edf4fb020910181f262d343b424950575e656c747b828990979ea5acb3bac1c8d0d7dee5ecf3fa01080f161d242c333a41484f565d646b727980888f969da4abb2b9c0c7ced5dce4ebf2f900070e151c232a313840474e555c636a71787f868d949ca3aab1b8bfc6cdd4dbe2e9f0f8ff060d141b222930373e454c545b626970777e858c939aa1a8b0b7bec5ccd3dae1e8eff6fd040c131a21282f363d444b525960686f767d848b9299a0a7aeb5bcc4cbd2d9e0e7eef5fc030a111820272e353c434a51585f666d747c838a91989fa6adb4bbc2c9d0d8dfe6edf4fb020910171e252c343b424950575e656c737a818890979ea5acb3bac1c8cfd6dde4ecf3fa01080f161d242b323940484f565d646b727980878e959ca4abb2b9c0c7ced5dce3eaf1f800070e151c232a31383f464d545c636a71787f868d949ba2a9b0b8bfc6cdd4dbe2e9f0f7fe050c141b222930373e454c535a616870777e858c939aa1a8afb6bdc4ccd3dae1e8eff6fd040b121920282f363d444b525960676e757c848b9299a0a7aeb5bcc3cad1d8e0e7eef5fc030a11181f262d343c434a51585f666d747b828990989fa6adb4bbc2c9d0d7dee5ecf4fb020910171e252c333a414850575e656c737a81888f969da4acb3bac1c8cfd6dde4ebf2f900080f161d242b323940474e555c646b727980878e959ca3aab1b8c0c7ced5dce3eaf1f8ff060d141c232a31383f464d545b626970787f868d949ba2a9b0b7bec5ccd4dbe2e9f0f7fe050c131a212830373e454c535a61686f767d848c939aa1a8afb6bdc4cbd2d9e0e8eff6fd040b121920272e353c444b525960676e757c838a9198a0a7aeb5bcc3cad1d8dfe6edf4fc030a11181f262d343b424950585f666d747b828990979ea5acb4bbc2c9d0d7dee5ecf3fa010810171e252c333a41484f565d646c737a81888f969da4abb2b9c0c8cfd6dde4ebf2f900070e151c242b323940474e555c636a717880878e959ca3aab1b8bfc6cdd4dce3eaf1f8ff060d141b222930383f464d545b626970777e858c949ba2a9b0b7bec5ccd3dae1e8f0f7fe050c131a21282f363d444c535a61686f767d848b9299a0a8afb6bdc4cbd2d9e0e7eef5fc040b121920272e353c434a515860676e757c838a91989fa6adb4bcc3cad1d8dfe6edf4fb020910181f262d343b424950575e656c747b828990979ea5acb3bac1c8d0d7dee5ecf3fa01080f161d242c333a41484f565d646b727980888f969da4abb2b9c0c7ced5dce4ebf2f900070e151c232a313840474e555c636a71787f868d949ca3aab1b8bfc6cdd4dbe2e9f0f8ff060d141b222930373e454c545b626970777e858c939aa1a8b0b7bec5ccd3dae1e8eff6fd040c131a21282f363d444b525960686f767d848b9299a0a7aeb5bcc4cbd2d9e0e7eef5fc030a111820272e353c434a51585f666d747c838a91989fa6adb4bbc2c9d0d8dfe6edf4fb020910171e252c343b424950575e656c737a818890979ea5acb3bac1c8cfd6dde4ecf3fa01080f161d242b323940484f565d646b727980878e959ca4abb2b9c0c7ced5dce3eaf1f800070e151c232a31", + "whole_ciphertext": "2a0ec302f289637828dab37afdfbad45e49589cf8aacbb52c77565813987ebe842cc61d48e26f986ab033bbb051e3ce3cf21a98c8b45780ab2dedb84875eb71d788c6631f5336245ffd3e6da192795968546189447d9ab9a5bdeaf8dabdf7bdb32a2ab6432722ed9b9be3e95343a7f37dfc83a7f958a10a3302c97e9882271e7dd4796dd3c28a0696690e324c5e6010d28cc7d01f9bad6627e0622e37cdd267f92faafc9f78866e0a33b37fdc7b67ff7de4db58247c933854f1eea1dff683594499773bceb425dbe892a6273901c6fe3d5bc98e6e54e8dbe8397dcfef280cfee55c5bf340c81ccf4d4fd47cd40c2cfefb6b9e47d30909a247b3d56d5150c290b5bec299a45371a77b569a4faf1a5971f2e49fcb82e9dbb28fc9d35d95cbcb865c88ca03bec82d943d517751bb3ef583a98c21acd66b59aa0e924a1fa319cc27456fb804cc1fd592c2b8e815be5ceebcfccdf3e3baa4fe730be985bcef0bada6fa654c56477be177a1b486b36bd90a81c6d88106278559ed7d8777163e8e9b179354826ac07a430e2f1f865c01c9c2610d20ebf5bad986fa5d7fc29fef56b2b4ec2de81cf72e8399ad234c88e2a62bda73b0bc3c94181942f03a2b6a433b5554252b510dd3f81ac69868d163433fa7326e5da4e2b1342a18f0948cad8b361b6fefbc8ac426261b3e9d043eeaed46b4ea41ad62282797d5e505cf3f7351470ceb8bf52568f49910570c7b16f32a37b52efa0f97326b9ad2f30a02c84f567514cb004d4782d31eb387a1c3c1d7f3ff184ad6e87f15ed8d93e7693a78b806b818673c229fc9a8d9ab2967903ceea324646bbd0c99412e3217c2f56a116f2d1b6031674bd3d33d197ad702a4812d4f4c4a17fafc7cdce1d4f7b491d0064999ef49f952a8d998baf77a65210ecec2381d8e13695318ca59b4602598fe0f58a2ee1bbd945dcbc0d571bc478d8401fffc1bd47218ffab2a966b247bdb93100ddd4612b18b82267eb3a8af6f8eeb668cbab3f31086dfc93815ade90d458234e88263535927d6c1400443a81cf72f31beddc0625d959de17f9cd8e9e719f430357e71684f59bbb2454b02533e7d15b8d28ccf54f55398c5dd193d12a14adff66cd9f46b2c3a80519cbc5d89c2abec0e80df70d2df4d9934287ec0640d1889c6f4f9f5b639f669753dbce78b72360440bb7821d9bfaef38bfc68dde2eb14f95637d2343537262bd3a677a2cde0031b286f76669e6d6fb62fca09eecc587d29292f0e09cc45e09b0f29672bf96d29b5620e1df94524c92c9b0b69eb16a8991117d00e977e92ff6c2dfae550e9049a88cb1d730318d8ee9b174c79b16bed6e5e1e2a8d21d179a84f2927fdd0e1feb35bbc796b15577e3586bea6276c42f0d34df34e667ef18c1f50bf5bc1a6db86f85b83b67d9162e0d64be141a36c19d9a13da8930f6faac38a000cf3623fa44ae2fb3ddad6345dabd9fad64248343580b92746794e8b30f91a5acba6f641155d7be863c6db494664895c23eb17988b1f6922181d9aa4ac2d06f07250c0e2e3cd1bd660b2dc3827def39f6113fcf6397f806f4d199f51e5f065d577ec20c009c52866b1b98dd2a4d68ad794978e24728986d1205f21c6ee2e32671646fbb49be468ee8b9bb9ca3e0defb483d54c114cc44f4e41240c8c68c357915c878e902e829329efbaf233f787c1b0cc65f9515b2e9565c892e1414d6e8d7bdb39af1f8e16f571f08936df36278a2afcbb44f852c2e1b6979e7c35abc486a610bbe97d5097a032b75a7996ad2b1e0e4dc102ad3f3be6254e9c0c3c9b5f706be25d742c710370568cbe5e54c346f5d65d49142e87d838f0f86a966d47ceb9bdfe0d893704bf70a6d95f5b221bd622d6145ae1875da4c0d53b7b43bd7b2d5804c243d5ac98b901bd949a1d91d2039e1f8865da69a54e349815fe92013104d8d3608c2cf378c9df81543760f646feba0b2e2e2bfbd869f9e19bc75b733efa15eb67b1cd53c2008bf3d1b6fc084497cba21a3f9741904353bbcab2bf8e1e4e86e1054c87bb0f7dca4b4bc52bf0120be8ec638d734712bab7776283c3a5c8fe621092a5ac4fac0d7a1bb754f7e02e9483bcbdc831335ebc96c36fef8f81333bc9986cf6bbe1c598c8c3e53b0f44968999f8eada87e646265dfcec8d31ae82ab611f390e1f7b1b72afb5576e8e682798714ab9172a0b96f0d739c7f21d25742040bfc129002d230fdc2d9f58c8ec9f8bfcadde8dd3d7259a5f263d94b79040ef9fca48c71991e4f4fbfd8c27be71707f632206e2f57eed446963fce6fbd1ee5591eec393512e46ec1a8a27b2aa3d5324f222d03db18df716b4166ef31f2a1d225a6a99d726919d3ac861ec3314a45988abcb21ac876ea1abb8848dafbaccbc1201f318c5911505c24d9edbae57bf58bb3c83a2ffee30cd35e8847e59c93f75b1e2247f4c12115bca558bd9b04f4c273b1001a301703cf749ed859de903c8c1c02f9e83f75c070287cff6112622f8a8847da1351e53f37728edfb0d85c0625aa8ea9265c97b02b89d1c3b74032e16bad92d3aaa3a37648f68bf1aaff61ec6841fc02d72af86aa7a10a280be213096f3fa73a958896a4c99be4e7c8e60b8c6e592bff01348c179643d29122eb773497c1e775837d2cfc7e1e34937e136f029a2a3ee852bcf37baae626e180e74b700c859b56b54251b48ffff05658488d90491815a16f8c5e8ca561a9344de7b9499771a528b1664663e5f29bf0cd6b909da922bc286b3b26ba621b5fa67ec387f974fb4bb9b9a28d450370dd1ebf419bd6c02b0a790e6b3fc44050c16d3e4007c81061079695095973870c1533f910e075bcd7d9776d0199aac5ceac2bdb6bc48e6a6dc24ecc7fa3480ec87880f628980786319ef91999d6f6443d817d68210d4ad178ed83d7ad0ef1d48544fac78ca8eb39beef1864cddab3ae099a6d5f908bb454370d78254c832684667e1f080e122b1b18dae91534428d5721c2fb58986b065375077edbba0977aa07083520c807b2402653e189be2b9c8916685040b4e66949675bbb026f50971522723a265077eea1c54772b189b5b271a7f2dd79cb392dbf70148cf944380a4bbe19fbd88919a92eef3377eed498f3bcf5d7f21037ac2ebcb0a3d91886bc1870d195727e344594439fb3f83982e1466efab1073db45b92120a49e70bc0e7a828a21d788fb2e6050102e90cf29e49dd4b1e78edf22e31c6dc33909e2d9d0a3582385760d417fdeaca8e0d31a51a9e1dc1c4300478210e5f5862da7f47263d8b018bf74f45f6fe326df42f78a1f468da714ddadd197266f4bdc3c3b69e6e060392a58eadbacf6bf93820aa13fadcc7c3122ba9370f536d6814e00bbc0a398f5ded3573b9fa0ed799c84b2c18d1dc098b524883a1d109deffe9a1b30e6d0717b10847db8859a647c89952fac8dda24acd57d1197ceb05c897213803b28b8f89158a9fc6dee50761319c19ad41bcd334ad3cbe2323c3ffc3626afadcc9a60dbd1e6749758ecf87745deaee02469843bbd7a7566d9f7e23f4c6519d1daff250efe83044492a55736e37551c805b8a83c0e7f337b670e3d84d8af22ab859cadeccc4d55cf6a9eed13aa3ac1bca813afbcacdbecdfe9553d14596df69f3da3134e43e9c090f8f7e204c6ee8bf3b1ad669513d548feb6c44bc2621346cedd01523b840e823c682579e2d0562a23f3de0641d7e0ea1a768484fb0a6ba8d850f9c4cf50ab92192bc74e21c70fe8049fd15a60169f47b6558f9e2de3aa26e18ee4d5922aa93159b2e4b2971f7a016cf1c680dbfee5bef0c8883bb78093272a86c117961b8472665568601320b5ebd02269d34eeb9b2f6854703dcba6ee791131d9ce674fef8b973afc0fcc6f25e37892a94353ef6940a71c1317aa10deb533f098e2527006c77522330e56fef913da5ae65c2b7d8c7f3be1307e0d5502edee9bacb42b58283555bc039bdf84a0f52ddba4ca742863786ec70e0973f03eece191e3eaa8c09a26b249c42087665380a1bf2cc2834193d407b26ad70d0ae2e9a43f0b3dea7b6326f70fca9070b3f3aa4be1b6433eb32437bd63b357e1f4148ad79ac47418cf3505343b1369d8e742db1f64701f1a75df58189b9667fdcacb1d261fe055be57b521f7798f570ccdb70938046956058cbb720f8a00a2a73fd61481ba9da6f7df7ddb9d33e2c58b679630163dddfd92729c95e60766966f1ef0a07c7b809c0c0a34ac8bcdfab513885dfe74db74d508d8fd391f6e986dae82af6d794ddc33ede0bb5c99271c8673cf6616691fbd9df298c93e79d210a1a4c2adf7e9c731c959efaab691dc89b704e175f2cfda88c0c2f38ff6352ee380f6a7713760c998eb2a66ccd81d142ae4509340879f4be282a643f9da2098cc95ee6ac661fd066f1a1b502834f5825db5744eeddb0f439659086ee7345e6d71215c16feec198907e48a19b42760b47dc7a1b7dae77724b2f7ca8e93008cddea794669e0b5d7b7e959af321e2f04916c3b12107c596a0856fb62a96ffb9b95c8c7697f79af3be336dfddf0fe90407bbfb35d850698cd180df2d09e5acf1d9f687c996e875d7a3f10aa29d89cbcbc03b7c95951b93ba959a175d1ce9b2edada2b715e503efc0da29e9154fe98c91ecf241f8bd6d5f3f2e4300d70cbf516a52d15d287a3721dd16d57ca29a92b2e09c937a22204ffb3f0ebc66584bfd70b4194b63997a4a90d8ba62c616dc19c794b625cb6ee5b1f890bc92d756c00c9caa105337e832991888b4cb0ee1f445dadae212c7ecf0e43c80aa3da6abdbf9ce101deb510dca8cb691f5c6f94648b9ed19d8feaf9fe26641dfc8be145d2ac0a84bfa5b3a4806b829d4c8fd88796093199cb08e2cec722faad44aef87819d69e999a6e0ff4f8e78dd7cdaeb03641cd5eebb210b2eef12d043db40b50246ad7a441574024acd3ff711acdfd8f85abf2f313c716dbb40c154da6958452fef6540a7c64e71647ce146e8120cfc98bd74a4162006d585ef289c2f4750920d9fe0cea926983553a3956d2c451da99d23f15c355a71f066f88f346e5ef6b83d029b61765ad3290a7b504fe1b8513164410b1ff8b073d4487393c0c7b3c1cbf3369fb718f41685c54f0ec483e5aa1e73ab92d948e543329c59e89ff4f4b255b9688cd597c53beb280fe0c0529e4b147401cc3a6da4d8728927fc71aeedc2aaa107d99ada4f489188d85bcb288e669f52c83fbfdd673627253278d69925b88b54343a4c4fa93aeb4b58cf76597d598df2373c2b96ccfde9c905dd03bc48130da42e877b9b6af8cd1d528a29a91576d3d2c20d1ae65c4f2f7492aea7895ef9908e9909d598f650d4bd3d4bf1dfdfe902806abb9f6661648bace8f119102b66366d261610de4463b2a324f8c7007d192c5fd829bd58fd268344ef05ec6c29db85aee26992e10f05a96c93b243373508e4cd9650182b24b615f803713a183d51191f708e9179b7ca7db8220917594f5547083f208f8f854623ca36963c48fee7c7f1776525871afb62a7488d4582538d028c18eec8bc0e7b01679131a05ad267802a102b9b9b99d96b5b8f53fa0f8cb19d43814a132d574c4907df0fe7076f91bf61dda818a746ac4425e0aecea670797faf821a70c040c9044d766144c021b0b9a8942041e12d6876e6aa4568566f3eaea607a06b5d770073b12a09a90bf6f9cffbbe74dbe2a0b60fb62a76ed1cc7e1a48f0b6a1b4e8879be59eeda03b46947afb3e30b84075b429752a5a4207169f500acb7b48a4125d5a21de431d00a338578198be20b08a59c3d6d3d723d9d266eb5dba5ebf94f450fb7a3869c58029cdd6218d4fd0807f9770eea155b88931b85c5b021fed8e6dd250e740cc4fa06d58d1adaaff1301bd5b390c6a59f0c0eb5cd65247e7c145ebb5e65fe8f7d49677d12d37ddc1199f21f68508db95983ee37d4f46cf2d28f575952038fb2e156134e6a935b7e7652594e97692cdc200164da6e2f339a6a5a06997bd25bdabc60d76f0fd32d09357559119c857522af1011598ad9ab63f8c0ceea527a59be0f0951b3b8a69f37a06078f6df825f94e6ad07b92856b33483eca5e4e356e574c728aa219cbc790cc34eb367e2883e09467fb50a8a590861a14add11416e5c27a51e00e3a2dab7b50d3b60f3b83d9e161932b94311c0d600a8689a8d9728855e569fc737b8eee40b427fd55011e099a63d7770460e9e9c413c1310590cc6f9a23e66c592f9c5fa72aaa74477ffb95cb2374917db0414c0cea2833343402696deeb08555f8e78c917ad53c67d4fa5dccd7dd927cdecf598ff0c096b77b4a61d607e9d3eaf48e06a5935d6d7946d716c60d07d2a4024548322da1a5250e788bb1240bc2bc67e594c893ccb594d1efdcb123de0290edd9f56c013e9201ceaef5fa77cbc2f6ba10456bb7f050dda281a528b4398b92a57d6064bf664280032baba7fdc8f9b5f1d869f11208c68e47108071ef7db3f2ededcd087191a4593bb58ff338f183402c46c2bad8e4a458fd42de859def9554fe7c216314ac6a3d4fb1a798bd14f821ae26a84741c4fbd710d6cccdc54a2769cf0bbcc7e6e8baf8faf2c4911fc7419965afd674698ff19147e7e436d7ba7fd25aa875eb9cb9c26c10b31183788b5b3b5e451aa0ecee00c177ae6d6f824c94993fc3ee16ac7bd960717866863aff4293872ffc798787c10ae418e68929248b976facf52990e68e94a2f6661d18ae444439fa314a396805f26bd62e04957098f4f6d10fd47e0eb64c34833075cb9e6dbf46934e1b51900f9feca34c9c6a7efd428c52f80b38800be4ee2e212c30f9e6e9bc21e0dc6e086e7f8e54bca54580687c4a2c53935bb2bab2837e36098224090be8ee78de8b1331b37344c8521f6d4e9f426735198d3d4941c2d25e1d62d99bb8b801bcc76a7f54eedd139f110b690443cbc24e6e42aab865d0e37bf7327a8df608365fafa16449ab13766e3f1c0c9ea9bcd09b814e458bb0432cab7e5394980dc6decd9cd98788250f25e4250d9c6f6d89fc7644f1fcf59018e55b5fc45dafb003f02cc279a96b8e202" +} diff --git a/seaweed-worker/crates/sw-client/Cargo.toml b/seaweed-worker/crates/sw-client/Cargo.toml new file mode 100644 index 0000000000..576b0c576b --- /dev/null +++ b/seaweed-worker/crates/sw-client/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "sw-client" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "SeaweedFS client library: filer gRPC metadata plane + volume HTTP data plane" + +[lib] +name = "sw_client" + +[dependencies] +anyhow.workspace = true +async-trait.workspace = true +bytes = "1" +log = "0.4" +prost.workspace = true +# System roots for volume servers behind public CAs, as Go's http client. +reqwest = { version = "0.12", default-features = false, features = ["multipart", "rustls-tls-native-roots"] } +rustls = { version = "0.23", default-features = false, features = ["ring"] } +thiserror = "2" +tokio.workspace = true +tonic.workspace = true +tonic-prost.workspace = true + +[dev-dependencies] +rustls-pemfile = "2" +tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] } + +[build-dependencies] +tonic-prost-build.workspace = true +protoc-bin-vendored = "3" + +[lints] +workspace = true diff --git a/seaweed-worker/crates/sw-client/build.rs b/seaweed-worker/crates/sw-client/build.rs new file mode 100644 index 0000000000..481d291e81 --- /dev/null +++ b/seaweed-worker/crates/sw-client/build.rs @@ -0,0 +1,24 @@ +fn main() -> Result<(), Box> { + // Use the protoc that ships with protoc-bin-vendored rather than a system + // one, so the build needs no package manager and always sees the same + // version. An explicit PROTOC still wins, for packagers supplying their own + // and for the lance crates, whose own build scripts read the same variable. + if std::env::var_os("PROTOC").is_none() { + // SAFETY: a build script's main runs single-threaded before anything + // else in this process, so no other thread can be reading the + // environment concurrently. + unsafe { + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?); + } + } + + // Compiled straight out of the Go tree, the way seaweed-volume already reads + // filer.proto, so the contract cannot drift from a vendored copy. + tonic_prost_build::configure() + .build_server(true) + .build_client(true) + .protoc_arg("--experimental_allow_proto3_optional") + .compile_protos(&["../../../weed/pb/filer.proto"], &["../../../weed/pb/"])?; + println!("cargo:rerun-if-changed=../../../weed/pb/filer.proto"); + Ok(()) +} diff --git a/seaweed-worker/crates/sw-client/src/cache.rs b/seaweed-worker/crates/sw-client/src/cache.rs new file mode 100644 index 0000000000..d61292afb5 --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/cache.rs @@ -0,0 +1,129 @@ +//! A small fileId-keyed whole-chunk cache for read prefetch. +//! +//! SeaweedFS needles are immutable for a given fileId (`,`), +//! so caching a chunk's bytes by fileId is coherence-safe — no invalidation is +//! ever needed, only capacity-bounded LRU eviction. This lets the read path +//! prefetch upcoming chunks in parallel (independent of the kernel's small, +//! single-chunk read upcalls) so sequential reads hit already-fetched bytes. + +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; + +/// Bounded LRU of fileId -> whole-chunk bytes. Not thread-safe on its own; wrap +/// in a `Mutex`. All ops are O(cache size) at worst (the LRU order is a deque), +/// which is fine for the few-hundred-entry caches used here. +pub struct ChunkCache { + map: HashMap>>, + order: VecDeque, // front = least-recently-used + bytes: usize, + cap: usize, +} + +impl ChunkCache { + pub fn new(cap_bytes: usize) -> Self { + Self { + map: HashMap::new(), + order: VecDeque::new(), + bytes: 0, + cap: cap_bytes, + } + } + + /// Fetch a cached chunk, marking it most-recently-used. + pub fn get(&mut self, file_id: &str) -> Option>> { + let v = self.map.get(file_id)?.clone(); + if let Some(pos) = self.order.iter().position(|x| x == file_id) { + let f = self.order.remove(pos).unwrap(); + self.order.push_back(f); + } + Some(v) + } + + pub fn contains(&self, file_id: &str) -> bool { + self.map.contains_key(file_id) + } + + /// Insert a chunk (no-op if empty, already present or larger than the whole + /// cap). Evicts least-recently-used entries until within capacity. + pub fn put(&mut self, file_id: String, data: Arc>) { + if self.map.contains_key(&file_id) { + return; + } + let len = data.len(); + if len == 0 || len > self.cap { + return; // a single chunk bigger than the whole cache: don't churn + } + self.map.insert(file_id.clone(), data); + self.order.push_back(file_id); + self.bytes += len; + while self.bytes > self.cap { + let Some(old) = self.order.pop_front() else { + break; + }; + if let Some(o) = self.map.remove(&old) { + self.bytes -= o.len(); + } + } + } + + #[cfg(test)] + pub fn len_bytes(&self) -> usize { + self.bytes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chunk(n: usize) -> Arc> { + Arc::new(vec![0u8; n]) + } + + #[test] + fn get_miss_then_hit() { + let mut c = ChunkCache::new(1024); + assert!(c.get("a").is_none()); + c.put("a".into(), chunk(100)); + assert_eq!(c.get("a").unwrap().len(), 100); + } + + #[test] + fn put_is_idempotent_on_bytes() { + let mut c = ChunkCache::new(1024); + c.put("a".into(), chunk(100)); + c.put("a".into(), chunk(100)); // duplicate ignored + assert_eq!(c.len_bytes(), 100); + } + + #[test] + fn lru_evicts_oldest_first() { + let mut c = ChunkCache::new(250); + c.put("a".into(), chunk(100)); + c.put("b".into(), chunk(100)); + // touch a so b is now least-recently-used + assert!(c.get("a").is_some()); + c.put("c".into(), chunk(100)); // bytes would be 300 > 250 -> evict LRU (b) + assert!(c.contains("a")); + assert!(!c.contains("b")); + assert!(c.contains("c")); + assert_eq!(c.len_bytes(), 200); + } + + #[test] + fn empty_chunks_are_not_cached() { + let mut c = ChunkCache::new(0); + for i in 0..10 { + c.put(i.to_string(), chunk(0)); + } + assert!(c.map.is_empty() && c.order.is_empty()); + } + + #[test] + fn oversized_chunk_not_cached() { + let mut c = ChunkCache::new(100); + c.put("big".into(), chunk(200)); + assert!(!c.contains("big")); + assert_eq!(c.len_bytes(), 0); + } +} diff --git a/seaweed-worker/crates/sw-client/src/filer.rs b/seaweed-worker/crates/sw-client/src/filer.rs new file mode 100644 index 0000000000..7cad70d1e7 --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/filer.rs @@ -0,0 +1,1287 @@ +//! Filer gRPC client — the SeaweedFS metadata plane. +//! +//! Only the filer endpoint is dialed; it proxies volume assignment/lookup, so +//! the master is never contacted directly. + +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity}; + +use crate::filer_pb::seaweed_filer_client::SeaweedFilerClient; +use crate::filer_pb::{ + AssignVolumeRequest, AtomicRenameEntryRequest, CacheRemoteObjectToLocalClusterRequest, + CreateEntryRequest, Entry, GetFilerConfigurationRequest, GetFilerConfigurationResponse, + KvGetRequest, KvPutRequest, ListEntriesRequest, LockRequest, LockResponse, + LookupDirectoryEntriesRequest, LookupDirectoryEntriesResponse, LookupDirectoryEntryRequest, + LookupVolumeRequest, PosixLockRange, PosixLockRequest, PosixLockResponse, StatisticsRequest, + StatisticsResponse, SubscribeMetadataRequest, SubscribeMetadataResponse, UnlockRequest, + UpdateEntryRequest, +}; + +/// Convert one `LookupDirectoryEntries` reply into the address book callers +/// execute against. A result carrying an error is unusable even when the Filer +/// attached an Entry (a chunk it could not place, a Volume it could not resolve +/// while Master was unreachable), so it fails the batch closed. Only +/// `found == false` without an error is a miss. +fn batch_lookup_response( + paths: &[(String, String)], + response: LookupDirectoryEntriesResponse, +) -> Result { + if response.results.len() != paths.len() { + bail!( + "LookupDirectoryEntries returned {} results for {} requests", + response.results.len(), + paths.len() + ); + } + + let mut results = Vec::with_capacity(paths.len()); + for (index, result) in response.results.into_iter().enumerate() { + if !result.error.is_empty() { + let (directory, name) = &paths[index]; + bail!( + "LookupDirectoryEntries result {index} ({directory}/{name}): {}", + result.error + ); + } + if result.found != result.entry.is_some() { + bail!("LookupDirectoryEntries result {index} has inconsistent found/entry state"); + } + results.push(BatchEntryLookup { + entry: result.entry, + log_pos: LogPos { + ts_ns: result.log_ts_ns, + signature: result.log_signature, + }, + }); + } + + let mut locations = HashMap::with_capacity(response.locations_map.len()); + for (volume_id, location_set) in response.locations_map { + let volume_id = volume_id + .parse::() + .with_context(|| format!("invalid volume id {volume_id:?} in batch lookup"))?; + let converted = location_set + .locations + .into_iter() + .map(|location| VolumeLocation { + url: location.url, + public_url: location.public_url, + grpc_port: location.grpc_port, + }) + .collect::>(); + if converted.is_empty() { + bail!("LookupDirectoryEntries returned no locations for volume {volume_id}"); + } + locations.insert(volume_id, converted); + } + + Ok(BatchEntryLookupResponse { + results, + locations, + read_auth: response.read_auth, + source: BatchLookupSource::ExactBatch, + }) +} + +/// A filer log position: the point in the metadata log that a served read or +/// an acknowledged write provably reflects. The filer stamps it under the same +/// per-path lock mutations hold across write and notify, so every event at or +/// below it is included and none above it are. +/// +/// `signature` names the filer whose clock produced `ts_ns`. Each filer stamps +/// with its own wall clock, so positions are only meaningful against events +/// that same filer logged. A zero signature means the position is not +/// attributable to any filer's clock — it came from an event rather than an +/// RPC — and such a position never fences anything. +/// +/// A zero `ts_ns` means the filer supplied no position (one older than the +/// field). Callers must treat that as "unknown", never as a low position. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct LogPos { + pub ts_ns: i64, + pub signature: i32, +} + +/// A physical volume location as returned by the Filer. Keep the advertised +/// gRPC port: runtime RDMA endpoint discovery must dial the exact volume server +/// and must not assume the historical HTTP+10000 port convention. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct VolumeLocation { + pub url: String, + pub public_url: String, + pub grpc_port: u32, +} + +/// One ordered exact-path result from a Filer batch lookup. +#[derive(Debug, Clone)] +pub struct BatchEntryLookup { + pub entry: Option, + pub log_pos: LogPos, +} + +/// How a batch lookup was resolved. +/// +/// A legacy per-entry fallback preserves ordinary metadata compatibility, but +/// it cannot prove that one Filer produced the complete direct-read address +/// book and its exact-FID capabilities. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum BatchLookupSource { + #[default] + Unknown, + ExactBatch, + LegacyPerEntry, +} + +/// Ordered metadata plus the physical address book needed for direct reads. +#[derive(Clone, Default)] +pub struct BatchEntryLookupResponse { + pub results: Vec, + pub locations: HashMap>, + /// Short-lived capabilities keyed by canonical FID. Empty when the Filer + /// does not sign reads or when a rolling-upgrade fallback was required. + pub read_auth: HashMap, + pub source: BatchLookupSource, +} + +impl std::fmt::Debug for BatchEntryLookupResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BatchEntryLookupResponse") + .field("result_count", &self.results.len()) + .field("volume_count", &self.locations.len()) + .field("read_auth_count", &self.read_auth.len()) + .field("source", &self.source) + .finish() + } +} + +impl VolumeLocation { + pub fn grpc_addr(&self) -> Result { + if self.url.is_empty() || self.grpc_port == 0 || self.grpc_port > u16::MAX as u32 { + bail!( + "invalid volume location url={:?} grpc_port={}", + self.url, + self.grpc_port + ); + } + let host = if self.url.starts_with('[') { + self.url + .split_once(']') + .map(|(host, _)| format!("{host}]")) + .ok_or_else(|| anyhow::anyhow!("invalid bracketed volume URL {}", self.url))? + } else { + self.url + .rsplit_once(':') + .map(|(host, _)| host.to_string()) + .unwrap_or_else(|| self.url.clone()) + }; + Ok(format!("{host}:{}", self.grpc_port)) + } +} + +impl LogPos { + /// Whether this position is known (the filer supplied one). + pub fn is_known(&self) -> bool { + self.ts_ns != 0 + } + + /// Whether the state at this position already accounts for an event at + /// `ts_ns` logged by the filers in `event_signatures`. + /// + /// Exact on a single filer: the stamp and the mutation share that filer's + /// per-path lock, so an event strictly below the stamp is in the value and + /// one at or above it is not. The bound is deliberately exclusive — the + /// stamp is taken before the read or write it fences, and a coarse clock + /// can log a later event within the same tick, so an equal timestamp is + /// ambiguous and must be applied rather than dropped. + /// + /// Across filers it is a heuristic, and knowingly so. Positions from + /// different clocks are not comparable, so a position must name the filer + /// that produced it and that filer must appear among the event's + /// signatures. That test is necessary but not sufficient: the serving filer + /// appends its own signature to every event it relays, peer events + /// included (`filer_grpc_server_sub_meta.go`), so a peer's timestamp can + /// still be judged against a local stamp. Under skew that can wrongly fence + /// a peer event, leaving a handle stale until the next event for the path. + /// Nothing in the signatures distinguishes the originating filer, so the + /// callers that cannot afford a wrong answer — orphaning a handle, or + /// replacing the object it holds — check entry identity instead, which + /// travels with the value and needs no clock. + pub fn covers(&self, ts_ns: i64, event_signatures: &[i32]) -> bool { + if !self.is_known() || ts_ns == 0 || ts_ns >= self.ts_ns { + return false; + } + self.signature != 0 && event_signatures.contains(&self.signature) + } +} + +/// One page of a directory listing (see [`FilerClient::list_entries_page`]). +#[derive(Debug, Clone, Default)] +pub struct EntryPage { + pub entries: Vec, + /// Name to resume after: the last entry of this page, or the caller's own + /// `start_from` if the page came back empty. Never rewinds, so feeding it + /// straight back always makes progress. + pub cursor: String, + /// Snapshot the filer echoed for this listing. Pass it back for the next + /// page — but it does not currently isolate the listing; see + /// [`FilerClient::list_entries_page`]. + pub snapshot_ts_ns: i64, + /// No entries follow this page. + pub eof: bool, +} + +/// What a paged listing must do to answer a request for a given entry index. +/// See [`seek_page`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Seek { + /// The page in hand covers it: serve from this offset within the page. + /// An offset at or past the page's length means the listing is exhausted. + Serve(usize), + /// Fetch the next page; the one in hand ends before the wanted index. + Advance, + /// Restart from the top; the wanted index precedes the page in hand. + Restart, +} + +/// Where a listing whose page holds `len` entries starting at offset `base` +/// (`eof` = that page runs to the end of the directory) stands relative to +/// entry `index`. +/// +/// Both front-ends page their readdir through this so the boundary rules have +/// one definition. The rules are asymmetric on purpose: paging only ever moves +/// forward, because the filer resumes a listing from a *name*, not an index — +/// so going back means starting over and paging up to the target again. +pub fn seek_page(base: usize, len: usize, eof: bool, index: usize) -> Seek { + if index < base { + return Seek::Restart; + } + // Past the page in hand: another page unless this one ended the directory, + // in which case serving an empty range is what reports EOF to the kernel. + if index >= base + len && !eof { + return Seek::Advance; + } + Seek::Serve(index - base) +} + +/// Result of assigning a volume for a new chunk write. +#[derive(Debug, Clone)] +pub struct Assignment { + pub file_id: String, + /// Volume server URL (host:port) to upload to. + pub url: String, + /// Per-fileId write JWT ("" if the cluster has no write signing key). + pub auth: String, + /// The path's storage rule wants the needle on disk before the volume + /// answers. + pub fsync: bool, +} + +/// Volume class hints passed to Seaweed's AssignVolume RPC. +/// +/// Data-plane transports should not make replica/placement decisions. They +/// receive the assigned `file_id + location` after this metadata-plane request. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AssignVolumeOptions { + pub collection: String, + pub replication: String, + /// Optional explicit volume-server placement (`host:http_port`). Empty + /// delegates placement to the master. + pub data_node: String, +} + +/// gRPC messages can be large (directory listings, fat chunk lists); raise the +/// default 4 MiB codec limits. +const MAX_MSG: usize = 256 * 1024 * 1024; + +/// A connected client to a single filer's gRPC endpoint. +/// +/// Cheap to clone — it shares the underlying HTTP/2 channel. +#[derive(Clone)] +pub struct FilerClient { + inner: SeaweedFilerClient, + addr: String, +} + +/// Cluster-wide settings fetched once at mount time. +#[derive(Debug, Clone)] +pub struct FilerConfig { + /// At-rest encryption. When true, chunks must be read whole (no server-side + /// range reads), which means read amplification. + pub cipher: bool, + /// Target chunk size in MiB for writes. + pub max_mb: u32, + /// This filer's signature. Metadata events we cause carry it, so the + /// `SubscribeMetadata` follower must skip them to avoid feedback loops. + pub signature: i32, + pub collection: String, + pub replication: String, + /// Directory holding the S3 buckets, `/buckets` unless configured. + pub dir_buckets: String, + pub version: String, + pub cluster_id: String, + pub filer_group: String, +} + +impl From for FilerConfig { + fn from(r: GetFilerConfigurationResponse) -> Self { + Self { + cipher: r.cipher, + max_mb: r.max_mb, + signature: r.signature, + collection: r.collection, + replication: r.replication, + dir_buckets: r.dir_buckets, + version: r.version, + cluster_id: r.cluster_id, + filer_group: r.filer_group, + } + } +} + +/// TLS for the filer's gRPC, as `grpc.client` and `grpc.ca` in security.toml +/// configure it for weed processes: the filer's certificate must chain to the +/// CA and name the host dialed, and this client presents its own. +#[derive(Clone)] +pub struct GrpcTls { + /// `grpc.ca` + pub ca_pem: Vec, + /// `grpc.client.cert` and `grpc.client.key` + pub cert_pem: Vec, + pub key_pem: Vec, + /// The name the filer's certificate is checked against, when it is not + /// the host in the filer's address. + pub server_name: Option, +} + +impl std::fmt::Debug for GrpcTls { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GrpcTls") + .field("server_name", &self.server_name) + .finish() + } +} + +impl GrpcTls { + fn config(&self, addr: &str) -> ClientTlsConfig { + crate::transport::install_crypto_provider(); + let host = addr + .rsplit_once(':') + .map_or(addr, |(host, _)| host) + .trim_matches(|c| c == '[' || c == ']'); + ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&self.ca_pem)) + .identity(Identity::from_pem(&self.cert_pem, &self.key_pem)) + .domain_name(self.server_name.clone().unwrap_or_else(|| host.to_string())) + } +} + +impl FilerClient { + /// Connect to a filer gRPC endpoint, e.g. `"10.0.0.1:18888"` (plaintext h2c). + pub async fn connect(addr: &str) -> Result { + Self::connect_with(addr, None).await + } + + /// [`connect`](Self::connect) over TLS when `tls` is given. + pub async fn connect_with(addr: &str, tls: Option<&GrpcTls>) -> Result { + let channel = Self::endpoint(addr, tls)? + .connect() + .await + .with_context(|| format!("connecting to filer {addr}"))?; + Ok(Self::with_channel(addr, channel)) + } + + /// [`connect`](Self::connect) whose channel connects on first use, so a + /// filer that is down right now is retried later instead of failing + /// startup. + pub fn connect_lazy(addr: &str) -> Result { + Self::connect_lazy_with(addr, None) + } + + /// [`connect_lazy`](Self::connect_lazy) over TLS when `tls` is given. + pub fn connect_lazy_with(addr: &str, tls: Option<&GrpcTls>) -> Result { + Ok(Self::with_channel( + addr, + Self::endpoint(addr, tls)?.connect_lazy(), + )) + } + + /// A client over a channel the caller dialed, for a TLS setup this crate + /// does not know, such as a server allowlist. + pub fn from_channel(addr: &str, channel: Channel) -> Self { + Self::with_channel(addr, channel) + } + + fn endpoint(addr: &str, tls: Option<&GrpcTls>) -> Result { + let scheme = if tls.is_some() { "https" } else { "http" }; + let endpoint = Endpoint::from_shared(format!("{scheme}://{addr}")) + .with_context(|| format!("invalid filer address {addr}"))? + .connect_timeout(Duration::from_secs(10)) + .tcp_keepalive(Some(Duration::from_secs(30))); + match tls { + Some(tls) => endpoint + .tls_config(tls.config(addr)) + .with_context(|| format!("TLS for filer {addr}")), + None => Ok(endpoint), + } + } + + fn with_channel(addr: &str, channel: Channel) -> Self { + Self { + inner: SeaweedFilerClient::new(channel) + .max_decoding_message_size(MAX_MSG) + .max_encoding_message_size(MAX_MSG), + addr: addr.to_string(), + } + } + + pub fn addr(&self) -> &str { + &self.addr + } + + /// Fetch cluster configuration (cipher, chunk size, signature, ...). + pub async fn get_configuration(&mut self) -> Result { + let resp = self + .inner + .get_filer_configuration(GetFilerConfigurationRequest {}) + .await + .context("GetFilerConfiguration RPC failed")?; + Ok(resp.into_inner().into()) + } + + /// Read a key from the filer's key-value store. `Ok(None)` when absent (the + /// filer answers a missing key with an empty value and no error, so an empty + /// stored value is indistinguishable from absent — callers store non-empty + /// values only). + pub async fn kv_get(&mut self, key: &[u8]) -> Result>> { + let req = KvGetRequest { key: key.to_vec() }; + let resp = self + .inner + .kv_get(req) + .await + .context("KvGet RPC failed")? + .into_inner(); + if !resp.error.is_empty() { + bail!("KvGet: {}", resp.error); + } + Ok((!resp.value.is_empty()).then_some(resp.value)) + } + + /// Write a key to the filer's key-value store. An empty `value` deletes the + /// key (the filer's KvPut semantics); deleting an absent key is not an error. + pub async fn kv_put(&mut self, key: &[u8], value: &[u8]) -> Result<()> { + let req = KvPutRequest { + key: key.to_vec(), + value: value.to_vec(), + }; + let resp = self + .inner + .kv_put(req) + .await + .context("KvPut RPC failed")? + .into_inner(); + if !resp.error.is_empty() { + bail!("KvPut: {}", resp.error); + } + Ok(()) + } + + /// Look up a single child `name` in `directory`. Returns `Ok(None)` when the + /// entry does not exist (the filer reports this as a generic error whose + /// message we recognise, not a gRPC NotFound status). + pub async fn lookup_entry(&mut self, directory: &str, name: &str) -> Result> { + Ok(self.lookup_entry_at(directory, name).await?.0) + } + + /// [`lookup_entry`](Self::lookup_entry) with the log position the returned + /// entry reflects, for callers that pin it (an open file handle) and must + /// later judge whether an event predates what they hold. + pub async fn lookup_entry_at( + &mut self, + directory: &str, + name: &str, + ) -> Result<(Option, LogPos)> { + let req = LookupDirectoryEntryRequest { + directory: directory.to_string(), + name: name.to_string(), + }; + match self.inner.lookup_directory_entry(req).await { + Ok(resp) => { + let resp = resp.into_inner(); + let pos = LogPos { + ts_ns: resp.log_ts_ns, + signature: resp.log_signature, + }; + Ok((resp.entry, pos)) + } + Err(status) if is_not_found(&status) => Ok((None, LogPos::default())), + Err(status) => Err(anyhow::Error::new(status) + .context(format!("LookupDirectoryEntry {directory}/{name}"))), + } + } + + /// Look up independent exact paths in one bounded RPC. + /// + /// Results preserve request order. Exact misses are represented by + /// `entry == None`; malformed entries and missing Volume locations fail the + /// whole batch closed. Older Filers fall back to the equivalent native + /// single-entry and Volume-location RPCs for rolling upgrades. + pub async fn lookup_entries_batch( + &mut self, + paths: &[(String, String)], + ) -> Result { + self.lookup_entries_batch_with_policy(paths, false).await + } + + /// Look up cache entries, mapping only a Master-confirmed retired Volume to + /// a miss. An unanswered lookup remains an error. + pub async fn lookup_cache_entries_batch( + &mut self, + paths: &[(String, String)], + ) -> Result { + self.lookup_entries_batch_with_policy(paths, true).await + } + + async fn lookup_entries_batch_with_policy( + &mut self, + paths: &[(String, String)], + unavailable_volume_is_miss: bool, + ) -> Result { + if paths.is_empty() { + return Ok(BatchEntryLookupResponse { + source: BatchLookupSource::ExactBatch, + ..Default::default() + }); + } + let requests = paths + .iter() + .map(|(directory, name)| LookupDirectoryEntryRequest { + directory: directory.clone(), + name: name.clone(), + }) + .collect(); + let response = match self + .inner + .lookup_directory_entries(LookupDirectoryEntriesRequest { + requests, + unavailable_volume_is_miss, + }) + .await + { + Ok(response) => response.into_inner(), + Err(status) if status.code() == tonic::Code::Unimplemented => { + return self.lookup_entries_legacy(paths).await; + } + Err(status) => { + return Err(anyhow::Error::new(status).context("LookupDirectoryEntries RPC failed")); + } + }; + batch_lookup_response(paths, response) + } + + async fn lookup_entries_legacy( + &mut self, + paths: &[(String, String)], + ) -> Result { + let mut results = Vec::with_capacity(paths.len()); + let mut volume_ids = HashSet::new(); + for (directory, name) in paths { + let (entry, log_pos) = self.lookup_entry_at(directory, name).await?; + if let Some(entry) = entry.as_ref() { + for chunk in &entry.chunks { + let fid = crate::read::file_id_string(chunk) + .context("legacy batch lookup entry chunk has no file id")?; + let (volume_id, _) = fid + .split_once(',') + .context("legacy batch lookup chunk has an invalid file id")?; + volume_ids.insert(volume_id.parse::().with_context(|| { + format!("legacy batch lookup has invalid volume id {volume_id:?}") + })?); + } + } + results.push(BatchEntryLookup { entry, log_pos }); + } + + let mut locations = HashMap::with_capacity(volume_ids.len()); + for volume_id in volume_ids { + let resolved = self.lookup_volume_locations(volume_id).await?; + if resolved.is_empty() { + bail!("legacy batch lookup returned no locations for volume {volume_id}"); + } + locations.insert(volume_id, resolved); + } + Ok(BatchEntryLookupResponse { + results, + locations, + read_auth: HashMap::new(), + source: BatchLookupSource::LegacyPerEntry, + }) + } + + /// Read one page of at most [`LIST_PAGE_SIZE`] entries from `directory`, + /// resuming after the name `start_from` (empty = from the beginning). + /// + /// There is deliberately no "list the whole directory" convenience wrapper: + /// a caller that collects every page holds the entire directory in memory, + /// which is gigabytes for a directory of millions of entries. Readdir is + /// served a batch at a time, so page it a batch at a time — keep the + /// returned [`EntryPage::cursor`] and drop the entries once emitted. + /// + /// Pass the previous page's [`EntryPage::snapshot_ts_ns`] back in (0 for a + /// new listing). + /// + /// Be careful what that buys: **the filer does not isolate the listing on + /// it.** `ListEntries` defaults the field to `time.Now()` when unset and + /// echoes it on the first response, but `StreamListDirectoryEntries` takes + /// no snapshot argument, so pages are read from live state. A paged listing + /// is therefore only as consistent as the directory is quiet: an entry + /// created before the cursor after that page was read is missed, and one + /// renamed from after the cursor to before it can be seen twice. Passing + /// the value back costs nothing and makes the client correct the day the + /// filer honours it. + pub async fn list_entries_page( + &mut self, + directory: &str, + start_from: &str, + snapshot_ts_ns: i64, + ) -> Result { + self.list_entries_page_prefixed(directory, "", start_from, snapshot_ts_ns) + .await + } + + /// [`list_entries_page`](Self::list_entries_page) of the entries whose + /// name starts with `prefix`. + pub async fn list_entries_page_prefixed( + &mut self, + directory: &str, + prefix: &str, + start_from: &str, + snapshot_ts_ns: i64, + ) -> Result { + let req = ListEntriesRequest { + directory: directory.to_string(), + prefix: prefix.to_string(), + start_from_file_name: start_from.to_string(), + inclusive_start_from: false, + limit: LIST_PAGE_SIZE, + snapshot_ts_ns, + ..Default::default() + }; + let mut stream = self + .inner + .list_entries(req) + .await + .with_context(|| format!("ListEntries {directory}"))? + .into_inner(); + + let mut entries: Vec = Vec::new(); + let mut snapshot = snapshot_ts_ns; + let mut cursor = start_from.to_string(); + while let Some(resp) = stream + .message() + .await + .with_context(|| format!("ListEntries {directory} stream"))? + { + // Pin the snapshot from the first response for page consistency. + if snapshot == 0 && resp.snapshot_ts_ns != 0 { + snapshot = resp.snapshot_ts_ns; + } + if let Some(entry) = resp.entry { + cursor.clear(); + cursor.push_str(&entry.name); + entries.push(entry); + } + } + + Ok(EntryPage { + // A short page means we've reached the end of the directory. + eof: entries.len() < LIST_PAGE_SIZE as usize, + entries, + cursor, + snapshot_ts_ns: snapshot, + }) + } + + /// Have the filer copy a remote-only entry's bytes onto the volume servers, + /// returning the entry with its chunks. + pub async fn cache_remote_object(&mut self, directory: &str, name: &str) -> Result { + let req = CacheRemoteObjectToLocalClusterRequest { + directory: directory.to_string(), + name: name.to_string(), + ..Default::default() + }; + self.inner + .cache_remote_object_to_local_cluster(req) + .await + .with_context(|| format!("CacheRemoteObjectToLocalCluster {directory}/{name}"))? + .into_inner() + .entry + .with_context(|| format!("the filer cached {directory}/{name} without an entry")) + } + + /// Resolve a volume id to full physical server locations. The filer proxies + /// the master, so clients never dial the master directly. + pub async fn lookup_volume_locations(&mut self, volume_id: u32) -> Result> { + let key = volume_id.to_string(); + let req = LookupVolumeRequest { + volume_ids: vec![key.clone()], + }; + let resp = self + .inner + .lookup_volume(req) + .await + .with_context(|| format!("LookupVolume {volume_id}"))? + .into_inner(); + let locations = resp + .locations_map + .get(&key) + .map(|locs| { + locs.locations + .iter() + .map(|location| VolumeLocation { + url: location.url.clone(), + public_url: location.public_url.clone(), + grpc_port: location.grpc_port, + }) + .collect() + }) + .unwrap_or_default(); + Ok(locations) + } + + /// Compatibility view for HTTP data-plane callers. + pub async fn lookup_volume(&mut self, volume_id: u32) -> Result> { + Ok(self + .lookup_volume_locations(volume_id) + .await? + .into_iter() + .map(|location| location.url) + .collect()) + } + + /// Assign a fileId + volume location for a new chunk of `size` bytes. + pub async fn assign_volume(&mut self, path: &str, size: u64) -> Result { + self.assign_volume_with_options(path, size, &AssignVolumeOptions::default()) + .await + } + + /// Assign a fileId + volume location for a new chunk using explicit volume + /// class hints from the mounted filer configuration. + pub async fn assign_volume_with_options( + &mut self, + path: &str, + size: u64, + options: &AssignVolumeOptions, + ) -> Result { + let req = assign_volume_request(path, size, options); + let resp = self + .inner + .assign_volume(req) + .await + .context("AssignVolume RPC failed")? + .into_inner(); + if !resp.error.is_empty() { + bail!("AssignVolume: {}", resp.error); + } + let url = resp.location.map(|l| l.url).unwrap_or_default(); + if resp.file_id.is_empty() || url.is_empty() { + bail!("AssignVolume returned no fileId/location"); + } + Ok(Assignment { + file_id: resp.file_id, + fsync: resp.fsync, + url, + auth: resp.auth, + }) + } + + /// Create an entry (upsert unless `o_excl`). `signature` tags the resulting + /// metadata event so our own subscription can skip it. `skip_check_parent` + /// bypasses the parent-directory existence check (used for hard links, whose + /// second name shares an already-validated parent). + pub async fn create_entry( + &mut self, + directory: &str, + entry: Entry, + signature: i32, + o_excl: bool, + skip_check_parent: bool, + ) -> Result { + let req = CreateEntryRequest { + directory: directory.to_string(), + entry: Some(entry), + o_excl, + signatures: vec![signature], + skip_check_parent_directory: skip_check_parent, + ..Default::default() + }; + let resp = self + .inner + .create_entry(req) + .await + .context("CreateEntry RPC failed")? + .into_inner(); + if !resp.error.is_empty() { + bail!("CreateEntry: {}", resp.error); + } + Ok(LogPos { + ts_ns: resp.log_ts_ns, + signature: resp.log_signature, + }) + } + + /// Replace an entry (send the whole mutated entry). + pub async fn update_entry( + &mut self, + directory: &str, + entry: Entry, + signature: i32, + ) -> Result { + let req = UpdateEntryRequest { + directory: directory.to_string(), + entry: Some(entry), + signatures: vec![signature], + ..Default::default() + }; + let resp = self + .inner + .update_entry(req) + .await + .context("UpdateEntry RPC failed")? + .into_inner(); + Ok(LogPos { + ts_ns: resp.log_ts_ns, + signature: resp.log_signature, + }) + } + + /// Delete an entry, optionally its data and recursively. + pub async fn delete_entry( + &mut self, + directory: &str, + name: &str, + is_recursive: bool, + delete_data: bool, + signature: i32, + ) -> Result<()> { + use crate::filer_pb::DeleteEntryRequest; + let req = DeleteEntryRequest { + directory: directory.to_string(), + name: name.to_string(), + is_recursive, + is_delete_data: delete_data, + ignore_recursive_error: false, + is_from_other_cluster: false, + signatures: vec![signature], + if_not_modified_after: 0, + ..Default::default() + }; + let resp = self + .inner + .delete_entry(req) + .await + .context("DeleteEntry RPC failed")? + .into_inner(); + if !resp.error.is_empty() { + bail!("DeleteEntry: {}", resp.error); + } + Ok(()) + } + + /// Atomically rename/move an entry (handles directories recursively). + pub async fn rename( + &mut self, + old_dir: &str, + old_name: &str, + new_dir: &str, + new_name: &str, + signature: i32, + ) -> Result<()> { + let req = AtomicRenameEntryRequest { + old_directory: old_dir.to_string(), + old_name: old_name.to_string(), + new_directory: new_dir.to_string(), + new_name: new_name.to_string(), + signatures: vec![signature], + ..Default::default() + }; + self.inner + .atomic_rename_entry(req) + .await + .context("AtomicRenameEntry RPC failed")?; + Ok(()) + } + + /// Fetch cluster-wide capacity/usage for a given volume class. Mirrors the + /// values `weed mount` reports via statfs (sizes are bytes). + pub async fn statistics( + &mut self, + replication: &str, + collection: &str, + ttl: &str, + ) -> Result { + let req = StatisticsRequest { + replication: replication.to_string(), + collection: collection.to_string(), + ttl: ttl.to_string(), + disk_type: String::new(), + }; + let resp = self + .inner + .statistics(req) + .await + .context("Statistics RPC failed")? + .into_inner(); + Ok(resp) + } + + /// Distributed advisory lock op via the filer's `PosixLock` service. `op` is + /// a `filer_pb::PosixLockOp` (TRY_LOCK / UNLOCK / GET_LK / KEEP_ALIVE); + /// `lock` is the byte range + owner; `keepalive` carries the full held set on + /// a KEEP_ALIVE re-assertion (empty otherwise). + pub async fn posix_lock( + &mut self, + key: &str, + op: i32, + lock: PosixLockRange, + keepalive: Vec, + ) -> Result { + let req = PosixLockRequest { + key: key.to_string(), + is_moved: false, + op, + lock: Some(lock), + locks: keepalive, + cooling_probe: false, + }; + let resp = self + .inner + .posix_lock(req) + .await + .context("PosixLock RPC failed")? + .into_inner(); + Ok(resp) + } + + /// Acquire/renew the filer's distributed lock named `name` for + /// `seconds_to_lock`. A fresh acquire passes an empty `renew_token`; a renew + /// passes the token from the previous response. The response's + /// `lock_host_moved_to` redirects to the filer that actually owns the key. + pub async fn distributed_lock( + &mut self, + name: &str, + seconds_to_lock: i64, + renew_token: &str, + owner: &str, + ) -> Result { + let req = LockRequest { + name: name.to_string(), + seconds_to_lock, + renew_token: renew_token.to_string(), + is_moved: false, + owner: owner.to_string(), + }; + let resp = self + .inner + .distributed_lock(req) + .await + .context("DistributedLock RPC failed")? + .into_inner(); + Ok(resp) + } + + /// Release the distributed lock `name` held under `renew_token`. + pub async fn distributed_unlock(&mut self, name: &str, renew_token: &str) -> Result<()> { + let req = UnlockRequest { + name: name.to_string(), + renew_token: renew_token.to_string(), + is_moved: false, + }; + self.inner + .distributed_unlock(req) + .await + .context("DistributedUnlock RPC failed")?; + Ok(()) + } + + /// Open a metadata-change subscription stream from `since_ns` under `/`. + /// + /// `client_id` MUST be non-zero (the filer's keep-alive check drops a zero + /// id and closes the stream immediately), and `client_epoch` must increase + /// on each reconnect for the same id (or the filer rejects it as a duplicate). + pub async fn subscribe( + &mut self, + client_name: &str, + client_id: i32, + client_epoch: i32, + since_ns: i64, + ) -> Result> { + let req = SubscribeMetadataRequest { + client_name: client_name.to_string(), + path_prefix: "/".to_string(), + since_ns, + client_id, + client_epoch, + ..Default::default() + }; + let stream = self + .inner + .subscribe_metadata(req) + .await + .context("SubscribeMetadata RPC failed")? + .into_inner(); + Ok(stream) + } +} + +fn assign_volume_request( + path: &str, + size: u64, + options: &AssignVolumeOptions, +) -> AssignVolumeRequest { + AssignVolumeRequest { + count: 1, + collection: options.collection.clone(), + replication: options.replication.clone(), + data_node: options.data_node.clone(), + path: path.to_string(), + expected_data_size: size, + ..Default::default() + } +} + +/// Entries fetched per `ListEntries` page. +const LIST_PAGE_SIZE: u32 = 1000; + +/// Whether `err` is a filer being unreachable or slow rather than refusing +/// the request, so that another filer may answer it. +pub fn is_unavailable(err: &anyhow::Error) -> bool { + err.chain().any(|cause| { + cause.downcast_ref::().is_some_and(|status| { + matches!( + status.code(), + tonic::Code::Unavailable + | tonic::Code::DeadlineExceeded + | tonic::Code::Cancelled + | tonic::Code::Aborted + ) || (status.code() == tonic::Code::Unknown + && status.message().contains("transport error")) + }) || cause.downcast_ref::().is_some() + }) +} + +/// SeaweedFS's filer returns a plain `ErrNotFound` Go error +/// ("filer: no entry is found in filer store"), which arrives as a gRPC status +/// with code `Unknown` rather than `NotFound` — so check the message too. +pub fn is_not_found(status: &tonic::Status) -> bool { + // Only the filer's exact missing-entry sentinel (filer_pb.ErrNotFound = + // "filer: no entry is found in filer store") or a proper NotFound code. A + // looser substring like "not found" would classify unrelated failures + // (e.g. "volume 7 not found") as a definitive miss — and the NFS resolve + // path attaches index-record DELETIONS to definitive misses, so a false + // positive would silently GC valid forwarding records. + status.code() == tonic::Code::NotFound || status.message().contains("no entry is found") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filer_pb::{FileChunk, Location, Locations, LookupDirectoryEntryResult}; + + fn batch_paths() -> Vec<(String, String)> { + vec![("/lm".to_string(), "key".to_string())] + } + + fn batch_entry(fid: &str) -> Entry { + Entry { + name: "key".to_string(), + chunks: vec![FileChunk { + file_id: fid.to_string(), + size: 4, + ..Default::default() + }], + ..Default::default() + } + } + + #[test] + fn batch_lookup_miss_is_found_false_without_an_error() { + let response = LookupDirectoryEntriesResponse { + results: vec![LookupDirectoryEntryResult::default()], + ..Default::default() + }; + let converted = batch_lookup_response(&batch_paths(), response).unwrap(); + assert!(converted.results[0].entry.is_none()); + assert!(converted.locations.is_empty()); + assert_eq!(converted.source, BatchLookupSource::ExactBatch); + } + + #[test] + fn batch_lookup_hit_carries_entry_and_locations() { + let response = LookupDirectoryEntriesResponse { + results: vec![LookupDirectoryEntryResult { + found: true, + entry: Some(batch_entry("9,01")), + ..Default::default() + }], + locations_map: HashMap::from([( + "9".to_string(), + Locations { + locations: vec![Location { + url: "volume-a:8080".to_string(), + grpc_port: 18_080, + ..Default::default() + }], + }, + )]), + ..Default::default() + }; + let converted = batch_lookup_response(&batch_paths(), response).unwrap(); + assert!(converted.results[0].entry.is_some()); + assert_eq!(converted.locations[&9][0].url, "volume-a:8080"); + } + + #[test] + fn batch_lookup_debug_redacts_read_capabilities() { + let response = BatchEntryLookupResponse { + read_auth: HashMap::from([("9,01".to_string(), "secret-exact-fid-token".to_string())]), + source: BatchLookupSource::ExactBatch, + ..Default::default() + }; + + let debug = format!("{response:?}"); + assert!(debug.contains("read_auth_count: 1")); + assert!(!debug.contains("secret-exact-fid-token")); + assert!(!debug.contains("9,01")); + } + + #[test] + fn batch_lookup_unresolved_volume_is_an_error_not_a_miss() { + // Master unreachable: the Filer keeps the Entry, reports why it could + // not place the volume, and leaves that volume's location set empty. + let response = LookupDirectoryEntriesResponse { + results: vec![LookupDirectoryEntryResult { + found: true, + entry: Some(batch_entry("9,01")), + error: "volume 9 has no locations: master unreachable".to_string(), + ..Default::default() + }], + locations_map: HashMap::from([("9".to_string(), Locations::default())]), + ..Default::default() + }; + let err = batch_lookup_response(&batch_paths(), response).unwrap_err(); + assert!( + err.to_string().contains("volume 9 has no locations"), + "{err}" + ); + } + + #[test] + fn batch_lookup_malformed_entry_keeps_its_error() { + let response = LookupDirectoryEntriesResponse { + results: vec![LookupDirectoryEntryResult { + found: true, + entry: Some(batch_entry("")), + error: "chunk has no file id".to_string(), + ..Default::default() + }], + ..Default::default() + }; + assert!(batch_lookup_response(&batch_paths(), response).is_err()); + } + + // A directory is listed by walking these decisions, so an off-by-one here is + // an entry served twice or skipped — a wrong `ls`, not a crash. Page size is + // irrelevant to the rules; these use a 10-entry page for legibility. + #[test] + fn seek_page_serves_from_within_the_page_in_hand() { + // Second page of a listing: entries 10..20 are in hand. + assert_eq!(seek_page(10, 10, false, 10), Seek::Serve(0)); + assert_eq!(seek_page(10, 10, false, 15), Seek::Serve(5)); + // The last entry of the page is still served from it, not paged past. + assert_eq!(seek_page(10, 10, false, 19), Seek::Serve(9)); + } + + #[test] + fn seek_page_advances_exactly_at_the_boundary() { + // One past the page in hand is the next page, not the last entry again. + assert_eq!(seek_page(10, 10, false, 20), Seek::Advance); + assert_eq!(seek_page(0, 1000, false, 1000), Seek::Advance); + // An empty window (a freshly opened listing) has nothing to serve. + assert_eq!(seek_page(0, 0, false, 0), Seek::Advance); + } + + #[test] + fn seek_page_stops_at_eof_instead_of_fetching_forever() { + // The final page ends the directory: reading past it must not ask the + // filer for another page, it must report the end. The offset lands at or + // past the page length, which the caller serves as an empty range. + assert_eq!(seek_page(10, 5, true, 15), Seek::Serve(5)); + assert_eq!(seek_page(10, 5, true, 99), Seek::Serve(89)); + // Within the final page, EOF changes nothing. + assert_eq!(seek_page(10, 5, true, 14), Seek::Serve(4)); + // An empty directory: the first page comes back empty and final. + assert_eq!(seek_page(0, 0, true, 0), Seek::Serve(0)); + } + + #[test] + fn seek_page_restarts_on_a_backward_seek() { + // rewinddir, or any seekdir to a cookie before the page in hand. The + // filer resumes by name, so there is no way back but from the top. + assert_eq!(seek_page(10, 10, false, 9), Seek::Restart); + assert_eq!(seek_page(10, 10, false, 0), Seek::Restart); + // EOF does not pin the window: a rewind past a finished listing restarts. + assert_eq!(seek_page(10, 5, true, 0), Seek::Restart); + } + + #[test] + fn assign_volume_request_carries_volume_class() { + let req = assign_volume_request( + "/models/checkpoint.bin", + 32 * 1024 * 1024, + &AssignVolumeOptions { + collection: "models".to_string(), + replication: "001".to_string(), + data_node: "volume-a:8080".to_string(), + }, + ); + + assert_eq!(req.count, 1); + assert_eq!(req.path, "/models/checkpoint.bin"); + assert_eq!(req.expected_data_size, 32 * 1024 * 1024); + assert_eq!(req.collection, "models"); + assert_eq!(req.replication, "001"); + assert_eq!(req.data_node, "volume-a:8080"); + } + + #[test] + fn volume_location_uses_advertised_grpc_port() { + let location = VolumeLocation { + url: "volume-a.example:8080".into(), + grpc_port: 18080, + ..Default::default() + }; + assert_eq!(location.grpc_addr().unwrap(), "volume-a.example:18080"); + } + + #[test] + fn volume_location_formats_ipv6_grpc_address() { + let location = VolumeLocation { + url: "[2001:db8::7]:8080".into(), + grpc_port: 18080, + ..Default::default() + }; + assert_eq!(location.grpc_addr().unwrap(), "[2001:db8::7]:18080"); + } + + #[test] + fn volume_location_rejects_missing_grpc_port() { + let location = VolumeLocation { + url: "volume-a.example:8080".into(), + ..Default::default() + }; + assert!(location.grpc_addr().is_err()); + } +} diff --git a/seaweed-worker/crates/sw-client/src/lib.rs b/seaweed-worker/crates/sw-client/src/lib.rs new file mode 100644 index 0000000000..3eceed004f --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/lib.rs @@ -0,0 +1,45 @@ +//! SeaweedFS client library. +//! +//! Speaks the filer gRPC metadata plane and the volume-server HTTP data plane, +//! and resolves files (lists of immutable chunks) into byte ranges. The vector +//! engine, the Lance maintenance worker and its CLI reach the cluster's +//! storage through it; the Enterprise repository's VFS client shares its API. + +/// Generated protobuf + gRPC client stubs for the SeaweedFS filer +/// (`package filer_pb;` in `weed/pb/filer.proto`). +pub mod filer_pb { + #![allow(clippy::all)] + #![allow(rustdoc::all)] + tonic::include_proto!("filer_pb"); +} + +mod cache; +pub mod filer; +pub mod location_cache; +pub mod read; +pub mod transport; +pub mod wbuf; +pub mod write; + +/// Upper bound on a single read's up-front allocation. `read_file` allocates the +/// requested span, so a malformed request with an absurd size could otherwise +/// trigger a huge allocation (OOM/DoS); a read span over this is rejected. This +/// is purely a per-read guard — file *size* is unbounded, since writes stream +/// chunk-by-chunk via [`wbuf::StreamingWriteBuffer`]. +pub const MAX_READ_BYTES: u64 = 1 << 30; /* 1 GiB */ + +pub use filer::{ + AssignVolumeOptions, Assignment, EntryPage, FilerClient, FilerConfig, GrpcTls, LogPos, Seek, + VolumeLocation, is_unavailable, seek_page, +}; +pub use location_cache::{DEFAULT_RETAIN_GENERATIONS, LocationCache}; +pub use read::{ + ChunkDecoder, ReadOptions, ResolvedChunks, VolumeReader, compact_shadowed, read_file, + read_file_with, read_resolved, resolve_chunks, +}; +pub use transport::{ + CommitAmbiguous, HttpTransport, HttpsTls, NotWritten, VolumeTransport, VolumeWriteReceipt, + install_crypto_provider, is_not_written, +}; +pub use wbuf::{SealPiece, StreamingWriteBuffer}; +pub use write::{Writer, flush_file, now_ns, now_unix, upload_pieces, upload_pieces_with_options}; diff --git a/seaweed-worker/crates/sw-client/src/location_cache.rs b/seaweed-worker/crates/sw-client/src/location_cache.rs new file mode 100644 index 0000000000..6505fe108b --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/location_cache.rs @@ -0,0 +1,303 @@ +//! Volume id -> volume server locations, cached for reads. +//! +//! Mirrors the Go client's vid map (`weed/wdclient/vid_map.go`): one flat map +//! whose entries remember the generation they were learned in. [`reset`] starts +//! a new generation when the metadata source has to be relearned, entries stay +//! readable while that happens, and any not relearned within +//! `retain_generations` resets are dropped. Nothing is chained, so there is no +//! history to walk and none to sever underneath a reader. +//! +//! [`reset`]: LocationCache::reset + +use std::collections::HashMap; +use std::sync::Mutex; + +/// How many resets a location survives without being relearned. Matches the Go +/// client's `DefaultVidMapCacheSize`. +pub const DEFAULT_RETAIN_GENERATIONS: u64 = 5; + +#[derive(Debug)] +struct Entry { + urls: Vec, + generation: u64, +} + +#[derive(Debug, Default)] +struct Inner { + entries: HashMap, + generation: u64, +} + +#[derive(Debug)] +pub struct LocationCache { + inner: Mutex, + retain_generations: u64, +} + +impl Default for LocationCache { + fn default() -> Self { + Self::new() + } +} + +impl LocationCache { + pub fn new() -> Self { + Self::with_retain_generations(DEFAULT_RETAIN_GENERATIONS) + } + + /// Keep locations for `retain_generations` resets after they were last + /// learned. Zero is raised to one: an entry always survives the reset that + /// immediately follows it, which is what keeps in-flight reads resolving. + pub fn with_retain_generations(retain_generations: u64) -> Self { + Self { + inner: Mutex::new(Inner::default()), + retain_generations: retain_generations.max(1), + } + } + + /// Locations for `vid`, or `None` when it is unknown or has none. There is + /// nothing older to fall back to: a volume's history lives in its own entry. + pub fn locations(&self, vid: u32) -> Option> { + let inner = self.inner.lock().unwrap(); + inner.entries.get(&vid).map(|entry| entry.urls.clone()) + } + + /// Record what the lookup returned for `vid`, replacing whatever an earlier + /// generation held. A volume that moved must not keep answering with the + /// server it moved off, and one with no locations left is dropped rather + /// than kept as an empty entry. + pub fn learn(&self, vid: u32, urls: Vec) { + let mut inner = self.inner.lock().unwrap(); + if urls.is_empty() { + inner.entries.remove(&vid); + return; + } + let generation = inner.generation; + inner.entries.insert(vid, Entry { urls, generation }); + } + + /// Forget `vid` so the next read looks it up again. + pub fn invalidate(&self, vid: u32) { + self.inner.lock().unwrap().entries.remove(&vid); + } + + /// Forget `vid` only while it still holds exactly `urls`, and report + /// whether it did. The counterpart of a read that failed against the + /// servers we had cached: a slow read failing against servers that a newer + /// lookup has already replaced must not throw the fresh entry away. + pub fn invalidate_stale(&self, vid: u32, urls: &[String]) -> bool { + let mut inner = self.inner.lock().unwrap(); + let stale = inner + .entries + .get(&vid) + .is_some_and(|entry| entry.urls == urls); + if stale { + inner.entries.remove(&vid); + } + stale + } + + /// Start a new generation, as when the metadata subscription reconnects and + /// what we were told may no longer hold. Entries stay readable until they + /// are relearned or fall out of the retained window. + pub fn reset(&self) { + let mut inner = self.inner.lock().unwrap(); + inner.generation += 1; + if inner.generation <= self.retain_generations { + return; + } + let oldest = inner.generation - self.retain_generations; + inner.entries.retain(|_, entry| entry.generation >= oldest); + } + + /// Number of volumes currently held, retained generations included. + pub fn len(&self) -> usize { + self.inner.lock().unwrap().entries.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn urls(urls: &[&str]) -> Vec { + urls.iter().map(|u| u.to_string()).collect() + } + + #[test] + fn unknown_volume_is_a_miss() { + let cache = LocationCache::new(); + assert_eq!(cache.locations(1), None); + } + + #[test] + fn learned_locations_are_served() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["h1:8080", "h2:8080"])); + assert_eq!(cache.locations(1), Some(urls(&["h1:8080", "h2:8080"]))); + } + + #[test] + fn empty_learn_is_a_definitive_miss() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["h1:8080"])); + cache.learn(1, Vec::new()); + assert_eq!( + cache.locations(1), + None, + "a volume known to have no locations must not serve what it had before" + ); + assert!(cache.is_empty(), "an emptied volume is dropped, not kept"); + } + + // The Go side's addLocation rule: the first write of a generation replaces + // the entry rather than merging into it. + #[test] + fn learn_replaces_earlier_generation() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["moved-from:8080"])); + cache.reset(); + cache.learn(1, urls(&["moved-to:8080"])); + assert_eq!(cache.locations(1), Some(urls(&["moved-to:8080"]))); + } + + #[test] + fn locations_survive_a_reset() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["h1:8080"])); + cache.reset(); + assert_eq!( + cache.locations(1), + Some(urls(&["h1:8080"])), + "a reset must not strand reads that are already in flight" + ); + } + + #[test] + fn entries_expire_after_the_retained_window() { + let retain = 2; + let cache = LocationCache::with_retain_generations(retain); + cache.learn(1, urls(&["h1:8080"])); + + for i in 0..retain { + cache.reset(); + assert!( + cache.locations(1).is_some(), + "location should still be retained after {} reset(s)", + i + 1 + ); + } + + cache.reset(); + assert_eq!(cache.locations(1), None); + assert!( + cache.is_empty(), + "expired entries must be dropped, not kept" + ); + } + + #[test] + fn relearning_restarts_the_window() { + let cache = LocationCache::with_retain_generations(2); + for _ in 0..5 { + cache.learn(1, urls(&["h1:8080"])); + cache.reset(); + } + assert!(cache.locations(1).is_some()); + } + + #[test] + fn zero_retention_still_survives_one_reset() { + let cache = LocationCache::with_retain_generations(0); + cache.learn(1, urls(&["h1:8080"])); + cache.reset(); + assert!(cache.locations(1).is_some()); + cache.reset(); + assert_eq!(cache.locations(1), None); + } + + #[test] + fn invalidate_drops_a_retained_entry() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["h1:8080"])); + cache.reset(); + cache.invalidate(1); + assert_eq!(cache.locations(1), None); + assert_eq!(cache.locations(2), None, "unrelated volumes are untouched"); + } + + #[test] + fn invalidate_stale_only_drops_what_the_caller_used() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["stale:8080"])); + + assert!(cache.invalidate_stale(1, &urls(&["stale:8080"]))); + assert_eq!(cache.locations(1), None); + + // A newer lookup has already replaced the entry the failed read used. + cache.learn(1, urls(&["fresh:8080"])); + assert!(!cache.invalidate_stale(1, &urls(&["stale:8080"]))); + assert_eq!( + cache.locations(1), + Some(urls(&["fresh:8080"])), + "a read failing against replaced servers must not drop the fresh entry" + ); + } + + #[test] + fn invalidate_stale_on_an_unknown_volume_is_a_no_op() { + let cache = LocationCache::new(); + assert!(!cache.invalidate_stale(1, &urls(&["h1:8080"]))); + } + + #[test] + fn invalidate_only_touches_its_own_volume() { + let cache = LocationCache::new(); + cache.learn(1, urls(&["h1:8080"])); + cache.learn(2, urls(&["h2:8080"])); + cache.invalidate(1); + assert_eq!(cache.locations(2), Some(urls(&["h2:8080"]))); + assert_eq!(cache.len(), 1); + } + + // A volume relearned every generation must always resolve, however many + // resets run concurrently with the readers. + #[test] + fn concurrent_resets_never_strand_a_live_volume() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let cache = Arc::new(LocationCache::with_retain_generations(2)); + cache.learn(1, urls(&["h1:8080"])); + let stop = Arc::new(AtomicBool::new(false)); + + let readers: Vec<_> = (0..4) + .map(|_| { + let cache = Arc::clone(&cache); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + assert!( + cache.locations(1).is_some(), + "live volume went missing during a reset" + ); + std::thread::yield_now(); + } + }) + }) + .collect(); + + for _ in 0..2_000 { + cache.reset(); + cache.learn(1, urls(&["h1:8080"])); + } + stop.store(true, Ordering::Relaxed); + for reader in readers { + reader.join().unwrap(); + } + } +} diff --git a/seaweed-worker/crates/sw-client/src/read.rs b/seaweed-worker/crates/sw-client/src/read.rs new file mode 100644 index 0000000000..94b15c1af6 --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/read.rs @@ -0,0 +1,1509 @@ +//! Data plane (read): resolve a file's chunks into visible byte intervals and +//! fetch the requested range from volume servers. +//! +//! A file is a list of immutable `FileChunk`s mapping logical byte ranges to +//! needles. Overlapping chunks are resolved latest-`modified_ts_ns`-wins per +//! byte; `is_chunk_manifest` chunks are expanded (fetched + gunzipped + +//! proto-decoded) recursively; gaps read as zeros (sparse files). Encrypted +//! chunks are decoded by the caller's [`ChunkDecoder`]. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result, bail}; +use prost::Message; + +use crate::cache::ChunkCache; +use crate::filer::FilerClient; +use crate::filer_pb::{Entry, FileChunk, FileChunkManifest, FileId}; +use crate::location_cache::LocationCache; +use crate::transport::{HttpTransport, VolumeTransport}; + +/// Max chunk fetches in flight per `read_file` call (the read's own chunks). +const READ_CONCURRENCY: usize = 16; +/// How many upcoming chunks to prefetch ahead of the current read position. +const PREFETCH_AHEAD: usize = 8; +/// Max background prefetch fetches in flight across all reads. +const PREFETCH_CONCURRENCY: usize = 8; +/// Whole-chunk cache capacity (bytes). Immutable-by-fileId, so coherence-safe. +const CHUNK_CACHE_CAP_BYTES: usize = 256 * 1024 * 1024; +/// A read whose offset is within this many bytes of a file's read frontier +/// counts as sequential (absorbs reordered concurrent readahead; rejects far +/// random jumps), enabling whole-chunk caching + prefetch-ahead. +const SEQ_TOLERANCE: u64 = 8 * 1024 * 1024; +/// Max distinct files tracked for sequential detection (bounds the map). +const STREAM_TRACK_CAP: usize = 4096; +/// Hysteresis depth: the mode (sequential/random) only flips after this many +/// consecutive contrary reads, so a single outlier doesn't thrash it. +const SEQ_HYSTERESIS: i8 = 3; + +/// One parallel chunk fetch's result: `(dst, range bytes)`. +type ChunkFetch = Result<(usize, Vec)>; + +/// Turns a chunk's stored bytes into the file's bytes, for chunks the filer +/// encrypted (`cipher_key`) or an S3 gateway encrypted (`sse_type`). Every +/// chunk a read touches goes through the decoder, manifest chunks included; +/// a plain chunk comes back as it is. +pub trait ChunkDecoder: Send + Sync { + /// Whether `chunk` decodes only as a whole, so a ranged read fetches the + /// entire needle. + fn needs_whole_chunk(&self, chunk: &FileChunk) -> bool; + /// Decodes `data`, the stored bytes of `chunk` starting `offset` bytes + /// into it. + fn decode(&self, chunk: &FileChunk, offset: u64, data: Vec) -> Result>; +} + +/// How [`read_resolved`] fetches. +#[derive(Clone, Default)] +pub struct ReadOptions { + /// Fetch exactly the bytes asked for, with no whole-needle caching and no + /// prefetch: for callers with their own cache, or random reads of large + /// files. + pub exact: bool, + pub decoder: Option>, +} + +fn encoded(chunk: &FileChunk) -> bool { + !chunk.cipher_key.is_empty() || chunk.sse_type != 0 +} + +/// Per-file read-access state for the sequential/random gate: the read frontier +/// (max end offset seen) plus a hysteresis counter (sequential when >= 0). +#[derive(Clone, Copy)] +struct StreamPattern { + frontier: u64, + counter: i8, +} + +/// Build the SeaweedFS fileId string for a chunk: `","`. +pub fn file_id_string(chunk: &FileChunk) -> Option { + if !chunk.file_id.is_empty() { + return Some(chunk.file_id.clone()); + } + chunk.fid.as_ref().map(fid_to_string) +} + +fn fid_to_string(fid: &FileId) -> String { + format!( + "{},{}", + fid.volume_id, + needle_id_cookie(fid.file_key, fid.cookie) + ) +} + +/// Mirror SeaweedFS `formatNeedleIdCookie`: 8-byte big-endian key followed by +/// 4-byte big-endian cookie, with leading zero bytes (only within the key) +/// stripped, then lowercase hex. +fn needle_id_cookie(key: u64, cookie: u32) -> String { + let mut bytes = [0u8; 12]; + bytes[0..8].copy_from_slice(&key.to_be_bytes()); + bytes[8..12].copy_from_slice(&cookie.to_be_bytes()); + let mut start = 0usize; + while start < 8 && bytes[start] == 0 { + start += 1; + } + let mut s = String::with_capacity((12 - start) * 2); + for b in &bytes[start..] { + s.push_str(&format!("{b:02x}")); + } + s +} + +fn chunk_volume_id(chunk: &FileChunk) -> Result { + if let Some(fid) = &chunk.fid { + return Ok(fid.volume_id); + } + if let Some((vid, _)) = chunk.file_id.split_once(',') { + return vid.parse::().context("parse volume id from file_id"); + } + bail!("chunk has neither fid nor file_id") +} + +/// Removes its single-flight gate from the `inflight` map when the fetch scope +/// exits, on every path. Without it, a failed or early-returning fetch would +/// leave its gate entry behind, growing the map by one per distinct fileId. +/// +/// It removes the entry only when the map still holds *this* gate (pointer +/// equality): a waiter can complete after a newer request has already installed +/// a fresh gate for the same fileId, and dropping that one would defeat its +/// single-flight (a redundant re-fetch, never incorrect — needles are immutable). +struct GateGuard { + inflight: Arc>>>>, + file_id: String, + gate: Arc>, +} + +impl Drop for GateGuard { + fn drop(&mut self) { + let mut inflight = self.inflight.lock().unwrap(); + if inflight + .get(&self.file_id) + .is_some_and(|g| Arc::ptr_eq(g, &self.gate)) + { + inflight.remove(&self.file_id); + } + } +} + +/// Reads chunk bytes from volume servers, caching volume-id → URL locations. +/// The actual byte transfer is delegated to a pluggable [`VolumeTransport`] +/// (HTTP by default; RDMA later). Cheap to clone and safe to share across +/// concurrent reads — the location cache is interior-mutable, so all methods +/// take `&self`. +#[derive(Clone)] +pub struct VolumeReader { + transport: Arc, + location_cache: Arc, + /// fileId -> whole-chunk bytes (immutable needles, coherence-safe). + chunks: Arc>, + /// Bounds background prefetch fetches across all in-flight reads. + prefetch_sem: Arc, + /// Per-file read-access pattern (first-chunk fileId -> frontier + hysteresis + /// counter), for the sequential-vs-random gate. Bounded by STREAM_TRACK_CAP. + streams: Arc>>, + /// Single-flight gates: fileId -> per-needle async lock. The prefetch re-issues + /// the next chunks on every read upcall; without this, each re-issue (and each + /// concurrent sub-read) races the in-flight fetch and re-GETs the whole 4 MiB + /// needle — measured ~5x over-fetch. The gate makes concurrent fetches of one + /// needle share a single GET. + inflight: Arc>>>>, +} + +impl Default for VolumeReader { + fn default() -> Self { + Self::new() + } +} + +impl VolumeReader { + pub fn new() -> Self { + Self::with_transport(Arc::new(HttpTransport::new())) + } + + /// Construct with a specific data-plane transport (e.g. an RDMA transport), + /// using the default whole-chunk cache capacity. + pub fn with_transport(transport: Arc) -> Self { + Self::with_transport_capped(transport, CHUNK_CACHE_CAP_BYTES) + } + + /// Construct with a specific transport and whole-chunk cache capacity (bytes). + /// The cap bounds resident read-cache memory: a browse/metadata-heavy mount can + /// run a much smaller cache than a streaming one. `0` disables caching. + pub fn with_transport_capped( + transport: Arc, + chunk_cache_cap_bytes: usize, + ) -> Self { + Self { + transport, + location_cache: Arc::new(LocationCache::new()), + chunks: Arc::new(Mutex::new(ChunkCache::new(chunk_cache_cap_bytes))), + prefetch_sem: Arc::new(tokio::sync::Semaphore::new(PREFETCH_CONCURRENCY)), + streams: Arc::new(Mutex::new(HashMap::new())), + inflight: Arc::new(Mutex::new(HashMap::new())), + } + } + + async fn locations(&self, client: &mut FilerClient, vid: u32) -> Result> { + if let Some(urls) = self.location_cache.locations(vid) { + return Ok(urls); + } + let urls = client.lookup_volume(vid).await?; + if urls.is_empty() { + bail!("no locations for volume {vid}"); + } + self.location_cache.learn(vid, urls.clone()); + Ok(urls) + } + + /// Locations currently cached for `vid`, without consulting the filer. + pub fn cached_locations(&self, vid: u32) -> Option> { + self.location_cache.locations(vid) + } + + /// Forget where a volume lives so the next read looks it up again. + pub fn invalidate_volume(&self, vid: u32) { + self.location_cache.invalidate(vid); + } + + /// Start a new location generation, for when the metadata source has to be + /// relearned (the filer subscription reconnecting, say). What we already + /// know stays readable until it is relearned or expires, so reads in flight + /// are not stranded. Counterpart of the Go client's resetVidMap. + pub fn reset_locations(&self) { + self.location_cache.reset(); + } + + /// Drop the cached locations for the volume behind `file_id` after a failed + /// fetch, so the next read relearns where the volume lives instead of + /// retrying servers it may have moved off. Only the `urls` the fetch + /// actually used are dropped: a slow read can fail long after another + /// lookup has learned where the volume moved to, and must not throw that + /// fresh answer away. + fn invalidate_after_failure(&self, file_id: &str, urls: &[String]) { + if let Some(vid) = file_id + .split(',') + .next() + .and_then(|v| v.parse::().ok()) + { + self.location_cache.invalidate_stale(vid, urls); + } + } + + /// Fetch a fileId from any of its volume servers, optionally a byte range, + /// via the configured transport. + async fn get( + &self, + urls: &[String], + file_id: &str, + range: Option<(u64, u64)>, + ) -> Result> { + self.transport_read(urls, file_id, range).await + } + + /// Every byte fetch goes through here so a failure always invalidates the + /// locations it was made against. + async fn transport_read( + &self, + urls: &[String], + file_id: &str, + range: Option<(u64, u64)>, + ) -> Result> { + let result = self.transport.read(urls, file_id, range).await; + if result.is_err() { + self.invalidate_after_failure(file_id, urls); + } + result + } + + /// Fetch a whole needle by fileId, served from / filling the chunk cache. + /// Needles are immutable per fileId, so a cache hit is always valid. The + /// mutex is only held to look up / insert — never across the network await. + async fn fetch_and_cache_whole(&self, urls: &[String], file_id: &str) -> Result>> { + if let Some(hit) = self.chunks.lock().unwrap().get(file_id) { + return Ok(hit); + } + // Single-flight: only one fetch per needle proceeds; others wait on the + // per-fileId gate, then hit the cache the winner just filled (instead of + // each issuing a duplicate GET of the whole 4 MiB needle). + let gate = self + .inflight + .lock() + .unwrap() + .entry(file_id.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone(); + // Drop the gate from the map on EVERY exit path (fill, cache-hit-after-lock, + // or a failed transport read) via this guard — an early `?` return here + // would otherwise leak one entry per distinct fileId that ever failed to + // fetch, growing `inflight` unboundedly on a flaky volume server. Waiters + // re-check and hit the just-filled cache; newcomers take the fast path. + let _gate_guard = GateGuard { + inflight: self.inflight.clone(), + file_id: file_id.to_string(), + gate: gate.clone(), + }; + let _g = gate.lock().await; + if let Some(hit) = self.chunks.lock().unwrap().get(file_id) { + return Ok(hit); + } + let bytes = self.transport_read(urls, file_id, None).await?; + let arc = Arc::new(bytes); + self.chunks + .lock() + .unwrap() + .put(file_id.to_string(), arc.clone()); + Ok(arc) + } + + /// Fetch exactly the needle's `[off, off+len)` bytes for one read. + /// - `cache_whole` (sequential reads): fetch + cache the whole needle, then + /// slice — so later sub-reads and prefetched chunks are served from memory. + /// - otherwise (random reads): serve from cache if already present, else + /// fetch only the requested range and do NOT cache — avoids over-reading a + /// whole 4 MB needle for a small random read. + async fn fetch_ranged( + &self, + urls: &[String], + file_id: &str, + off: usize, + len: usize, + cache_whole: bool, + ) -> Result> { + if cache_whole { + let whole = self.fetch_and_cache_whole(urls, file_id).await?; + let end = (off + len).min(whole.len()); + return Ok(if off < end { + whole[off..end].to_vec() + } else { + Vec::new() + }); + } + if let Some(whole) = self.chunks.lock().unwrap().get(file_id) { + let end = (off + len).min(whole.len()); + return Ok(if off < end { + whole[off..end].to_vec() + } else { + Vec::new() + }); + } + self.transport_read(urls, file_id, Some((off as u64, len as u64))) + .await + } + + fn copy_range_from_whole(whole: &[u8], off: usize, len: usize, dst: &mut [u8]) -> usize { + let end = (off + len).min(whole.len()); + if off >= end { + return 0; + } + let n = (end - off).min(dst.len()); + dst[..n].copy_from_slice(&whole[off..off + n]); + n + } + + fn copy_cached_range( + &self, + file_id: &str, + off: usize, + len: usize, + dst: &mut [u8], + ) -> Option { + let whole = self.chunks.lock().unwrap().get(file_id)?; + Some(Self::copy_range_from_whole(&whole, off, len, dst)) + } + + async fn fetch_ranged_into( + &self, + urls: &[String], + file_id: &str, + off: usize, + len: usize, + cache_whole: bool, + dst: &mut [u8], + ) -> Result { + if cache_whole { + let whole = self.fetch_and_cache_whole(urls, file_id).await?; + return Ok(Self::copy_range_from_whole(&whole, off, len, dst)); + } + if let Some(n) = self.copy_cached_range(file_id, off, len, dst) { + return Ok(n); + } + let bytes = self + .transport_read(urls, file_id, Some((off as u64, len as u64))) + .await?; + let n = bytes.len().min(dst.len()); + dst[..n].copy_from_slice(&bytes[..n]); + Ok(n) + } + + /// Update a file's read-access pattern and report whether reads should be + /// treated as sequential — frontier + tolerance, smoothed by hysteresis: + /// - `near` = this read starts within SEQ_TOLERANCE of the read frontier + /// (the window absorbs reordered/concurrent readahead, unlike strict + /// contiguity, while still rejecting far random jumps); + /// - the hysteresis counter ramps within [-SEQ_HYSTERESIS, SEQ_HYSTERESIS], + /// so a lone outlier read doesn't flip the mode; + /// - sequential = counter >= 0. A new file starts at frontier 0 / counter 0, + /// so a stream from the start is sequential and a far first read is random. + fn note_sequential(&self, key: &str, offset: u64, end: u64) -> bool { + let mut m = self.streams.lock().unwrap(); + if !m.contains_key(key) && m.len() >= STREAM_TRACK_CAP { + m.clear(); // crude bound; transient streams just re-confirm next read + } + let p = m.entry(key.to_string()).or_insert(StreamPattern { + frontier: 0, + counter: 0, + }); + let near = offset.abs_diff(p.frontier) <= SEQ_TOLERANCE; + p.frontier = p.frontier.max(end); + if near { + p.counter = (p.counter + 1).min(SEQ_HYSTERESIS); + } else { + p.counter = (p.counter - 1).max(-SEQ_HYSTERESIS); + } + p.counter >= 0 + } + + /// Spawn bounded background prefetches of upcoming whole chunks into the + /// cache. Fire-and-forget: errors are ignored (the real read will retry), + /// already-cached chunks are skipped, and the semaphore caps fan-out. + fn prefetch(&self, ahead: Vec<(String, Vec)>) { + for (file_id, urls) in ahead { + if self.chunks.lock().unwrap().contains(&file_id) { + continue; + } + let volumes = self.clone(); + let sem = self.prefetch_sem.clone(); + tokio::spawn(async move { + let Ok(_permit) = sem.acquire_owned().await else { + return; + }; + let _ = volumes.fetch_and_cache_whole(&urls, &file_id).await; + }); + } + } +} + +/// A non-overlapping visible byte interval `[start, stop)` of the file, backed +/// by part of one chunk. +#[derive(Clone)] +struct Visible { + start: i64, + stop: i64, + file_id: String, + vid: u32, + offset_in_chunk: i64, + has_cipher: bool, + /// Index of the backing chunk in the resolved data chunks. + chunk: usize, +} + +/// Recursively expand manifest chunks into plain data chunks. +async fn expand_chunks( + client: &mut FilerClient, + volumes: &VolumeReader, + chunks: &[FileChunk], + decoder: Option<&dyn ChunkDecoder>, +) -> Result> { + let mut data = Vec::new(); + for chunk in chunks { + if !chunk.is_chunk_manifest { + data.push(chunk.clone()); + continue; + } + let fid = file_id_string(chunk).context("manifest chunk missing fileId")?; + let vid = chunk_volume_id(chunk)?; + let urls = volumes.locations(client, vid).await?; + // The volume server transparently decompresses on a plain GET (we don't + // send Accept-Encoding: gzip), so the bytes are the raw manifest proto, + // unless the filer encrypted them. + let raw = volumes.get(&urls, &fid, None).await?; + let raw = match decoder { + Some(decoder) => decoder.decode(chunk, 0, raw)?, + None if encoded(chunk) => { + bail!("encrypted manifest chunk {fid} needs a decoder") + } + None => raw, + }; + let manifest = FileChunkManifest::decode(raw.as_slice()) + .with_context(|| format!("decode manifest {fid}"))?; + let sub = Box::pin(expand_chunks(client, volumes, &manifest.chunks, decoder)).await?; + data.extend(sub); + } + Ok(data) +} + +/// Resolve data chunks into non-overlapping visible intervals (latest write wins). +fn resolve_visibles(chunks: &[FileChunk]) -> Vec { + if let Some(visibles) = resolve_visibles_ordered_non_overlapping(chunks) { + return visibles; + } + + let mut order: Vec = (0..chunks.len()).collect(); + order.sort_by_key(|&i| (chunks[i].modified_ts_ns, chunks[i].offset)); + + let mut visibles: Vec = Vec::new(); + for &i in &order { + let c = &chunks[i]; + let start = c.offset; + let stop = c.offset + c.size as i64; + if stop <= start { + continue; + } + let Some(file_id) = file_id_string(c) else { + continue; + }; + let Ok(vid) = chunk_volume_id(c) else { + continue; + }; + + // This chunk is newer than everything already placed, so it overwrites + // any overlapping region: trim existing intervals around [start, stop). + let mut next: Vec = Vec::with_capacity(visibles.len() + 1); + for v in visibles.drain(..) { + if v.stop <= start || v.start >= stop { + next.push(v); + continue; + } + if v.start < start { + let mut left = v.clone(); + left.stop = start; + next.push(left); + } + if v.stop > stop { + let mut right = v.clone(); + right.offset_in_chunk += stop - v.start; + right.start = stop; + next.push(right); + } + // middle is overwritten by the newer chunk + } + next.push(Visible { + start, + stop, + file_id, + vid, + offset_in_chunk: 0, + has_cipher: !c.cipher_key.is_empty(), + chunk: i, + }); + next.sort_by_key(|v| v.start); + visibles = next; + } + visibles +} + +fn resolve_visibles_ordered_non_overlapping(chunks: &[FileChunk]) -> Option> { + let mut visibles = Vec::with_capacity(chunks.len()); + let mut last_stop = i64::MIN; + + for (i, c) in chunks.iter().enumerate() { + let start = c.offset; + let stop = c.offset + c.size as i64; + if stop <= start || start < last_stop { + return None; + } + let file_id = file_id_string(c)?; + let vid = chunk_volume_id(c).ok()?; + visibles.push(Visible { + start, + stop, + file_id, + vid, + offset_in_chunk: 0, + has_cipher: !c.cipher_key.is_empty(), + chunk: i, + }); + last_stop = stop; + } + + Some(visibles) +} + +/// Drop chunks that contribute no visible bytes — i.e. ones fully overwritten by +/// newer (higher `modified_ts_ns`) chunks. Cheap, metadata-only: no data is read +/// or moved. Used after a streaming flush appends overlay chunks, so a full +/// rewrite leaves the old chunks unreferenced (reclaimable) instead of leaking +/// them. Conservatively a no-op when manifest chunks are present (their offsets +/// span sub-chunks, so naive overlap resolution doesn't apply). +pub fn compact_shadowed(chunks: Vec) -> Vec { + if chunks.len() < 2 || chunks.iter().any(|c| c.is_chunk_manifest) { + return chunks; + } + let live: std::collections::HashSet = resolve_visibles(&chunks) + .into_iter() + .map(|v| v.file_id) + .collect(); + chunks + .into_iter() + .filter(|c| file_id_string(c).map(|f| live.contains(&f)).unwrap_or(true)) + .collect() +} + +/// A file's chunks with manifests expanded and overlaps resolved, reusable +/// across reads of one entry. +#[derive(Default)] +pub struct ResolvedChunks { + chunks: Vec, + visibles: Vec, +} + +impl ResolvedChunks { + /// Resolves chunks holding no manifests, without touching the network. + pub fn from_data_chunks(chunks: Vec) -> Result { + if let Some(manifest) = chunks.iter().find(|c| c.is_chunk_manifest) { + bail!( + "chunk {} is a manifest; use resolve_chunks", + file_id_string(manifest).unwrap_or_default() + ); + } + let visibles = resolve_visibles(&chunks); + Ok(Self { chunks, visibles }) + } + + /// One past the last byte any chunk covers. + pub fn end(&self) -> u64 { + self.chunks + .iter() + .map(|c| c.offset.max(0) as u64 + c.size) + .max() + .unwrap_or(0) + } +} + +/// Expands the manifests among `chunks` and resolves overlaps. Encrypted +/// manifests are decoded with `options.decoder`. +pub async fn resolve_chunks( + client: &mut FilerClient, + volumes: &VolumeReader, + chunks: &[FileChunk], + options: &ReadOptions, +) -> Result { + let chunks = expand_chunks(client, volumes, chunks, options.decoder.as_deref()).await?; + let visibles = resolve_visibles(&chunks); + Ok(ResolvedChunks { chunks, visibles }) +} + +/// Read `[offset, offset+size)` of a file, clamped to its size. Gaps read as +/// zeros. Returns at most `size` bytes (fewer at EOF). +pub async fn read_file( + client: &mut FilerClient, + volumes: &VolumeReader, + entry: &Entry, + offset: u64, + size: u64, +) -> Result> { + read_file_with( + client, + volumes, + entry, + offset, + size, + &ReadOptions::default(), + ) + .await +} + +/// [`read_file`] with [`ReadOptions`]. +pub async fn read_file_with( + client: &mut FilerClient, + volumes: &VolumeReader, + entry: &Entry, + offset: u64, + size: u64, + options: &ReadOptions, +) -> Result> { + let resolved = resolve_chunks(client, volumes, &entry.chunks, options).await?; + read_resolved(client, volumes, &resolved, entry, offset, size, options).await +} + +/// One chunk's part of a read: `off`/`len` within the needle, `dst` in the +/// output. +struct Fetch { + dst: usize, + chunk: FileChunk, + file_id: String, + urls: Vec, + off: usize, + len: usize, + /// The decoder needs the whole needle. + whole: bool, +} + +/// Fetches one piece of a chunk, decoding it when the read has a decoder. +async fn fetch_piece( + volumes: &VolumeReader, + f: &Fetch, + cache_whole: bool, + decoder: Option<&dyn ChunkDecoder>, +) -> Result> { + if f.whole { + let raw = if cache_whole { + volumes + .fetch_and_cache_whole(&f.urls, &f.file_id) + .await? + .to_vec() + } else { + volumes.transport_read(&f.urls, &f.file_id, None).await? + }; + let data = decoder + .context("a whole-chunk fetch needs a decoder")? + .decode(&f.chunk, 0, raw)?; + let end = (f.off + f.len).min(data.len()); + return Ok(if f.off < end { + data[f.off..end].to_vec() + } else { + Vec::new() + }); + } + let raw = volumes + .fetch_ranged(&f.urls, &f.file_id, f.off, f.len, cache_whole) + .await?; + match decoder { + Some(decoder) => decoder.decode(&f.chunk, f.off as u64, raw), + None => Ok(raw), + } +} + +/// Read `[offset, offset+size)` of `entry` through its resolved chunks: at +/// most `size` bytes, fewer at the end of the file, gaps as zeros. +pub async fn read_resolved( + client: &mut FilerClient, + volumes: &VolumeReader, + resolved: &ResolvedChunks, + entry: &Entry, + offset: u64, + size: u64, + options: &ReadOptions, +) -> Result> { + let file_size = entry + .attributes + .as_ref() + .map_or(0, |a| a.file_size) + .max(resolved.end()) + .max(entry.content.len() as u64); + if offset >= file_size || size == 0 { + return Ok(Vec::new()); + } + let end = offset.saturating_add(size).min(file_size); + if end - offset > crate::MAX_READ_BYTES { + bail!( + "read span too large ({} bytes, cap {})", + end - offset, + crate::MAX_READ_BYTES + ); + } + let want = (end - offset) as usize; + + // Small files store their bytes inline on the entry. + if !entry.content.is_empty() { + let content = &entry.content; + let s = (offset as usize).min(content.len()); + let e = (end as usize).min(content.len()); + let mut out = vec![0u8; want]; + out[..(e - s)].copy_from_slice(&content[s..e]); + return Ok(out); + } + + let visibles = &resolved.visibles; + + let r_start = offset as i64; + let r_stop = end as i64; + + // Detect sequential vs random access for this file (keyed by its first + // chunk's stable fileId) to gate prefetch + whole-chunk caching. + let stream_key = visibles.first().map(|v| v.file_id.clone()); + let sequential = !options.exact + && stream_key + .as_deref() + .map(|k| volumes.note_sequential(k, offset, end)) + .unwrap_or(false); + + // Pass 1: descriptors for the chunks this read needs. Volume location lookups + // need `&mut client`, so they're resolved serially here (cached after first); + // `off` is the byte offset WITHIN the needle, `dst` the offset in `out`. + let mut fetches: Vec = Vec::new(); + for v in visibles { + let cs = v.start.max(r_start); + let ce = v.stop.min(r_stop); + if cs >= ce { + continue; + } + let chunk = &resolved.chunks[v.chunk]; + let whole = match &options.decoder { + Some(decoder) => decoder.needs_whole_chunk(chunk), + None if encoded(chunk) => bail!("encrypted chunk {} needs a decoder", v.file_id), + None => false, + }; + let urls = volumes.locations(client, v.vid).await?; + let dst = (cs - r_start) as usize; + let len = (ce - cs) as usize; + fetches.push(Fetch { + dst, + chunk: chunk.clone(), + file_id: v.file_id.clone(), + urls, + off: (v.offset_in_chunk + (cs - v.start)) as usize, + len, + whole, + }); + } + // One chunk covering the whole read, the common shape of an exact read + // of a large file: hand its bytes back without a buffer to zero and copy + // into. + if options.exact && fetches.len() == 1 && fetches[0].dst == 0 && fetches[0].len == want { + let mut bytes = + fetch_piece(volumes, &fetches[0], false, options.decoder.as_deref()).await?; + bytes.resize(want, 0); + return Ok(bytes); + } + + // Zero-initialised so gaps between chunks, sparse regions, and short-read + // tails read back as zero. `vec![0u8; want]` lowers to `alloc_zeroed`, so a + // large buffer is backed by pre-zeroed OS pages (no memset) that the fetches + // below overwrite in place. + let mut out = vec![0u8; want]; + + // Prefetch-ahead ONLY for sequential streams: background whole-chunk fetches + // of the next PREFETCH_AHEAD chunks past this read so the following (small, + // single-chunk) upcalls hit the cache. Skipped for random access. + if sequential { + let mut ahead: Vec<(String, Vec)> = Vec::new(); + for v in visibles { + if v.start >= r_stop && !v.has_cipher { + let urls = volumes.locations(client, v.vid).await?; + ahead.push((v.file_id.clone(), urls)); + if ahead.len() >= PREFETCH_AHEAD { + break; + } + } + } + volumes.prefetch(ahead); + } + + // Pass 2: fetch each chunk's requested range (cache-aware), bounded by + // READ_CONCURRENCY. Sequential reads fetch+cache whole needles (so sub-reads + // and prefetched chunks hit memory); random reads fetch only the exact range. + if fetches.len() <= 1 { + if let Some(f) = fetches.into_iter().next() { + let dst_end = (f.dst + f.len).min(want); + if options.decoder.is_none() { + volumes + .fetch_ranged_into( + &f.urls, + &f.file_id, + f.off, + f.len, + sequential, + &mut out[f.dst..dst_end], + ) + .await?; + } else { + let bytes = + fetch_piece(volumes, &f, sequential, options.decoder.as_deref()).await?; + let n = bytes.len().min(dst_end - f.dst); + out[f.dst..f.dst + n].copy_from_slice(&bytes[..n]); + } + } + } else { + let sem = Arc::new(tokio::sync::Semaphore::new(READ_CONCURRENCY)); + let mut set: tokio::task::JoinSet = tokio::task::JoinSet::new(); + for f in fetches { + let volumes = volumes.clone(); + let sem = sem.clone(); + let decoder = options.decoder.clone(); + set.spawn(async move { + let _permit = sem.acquire_owned().await.expect("read semaphore closed"); + let bytes = fetch_piece(&volumes, &f, sequential, decoder.as_deref()).await?; + Ok((f.dst, bytes)) + }); + } + while let Some(joined) = set.join_next().await { + let (dst, bytes) = joined.context("chunk fetch task panicked")??; + let n = bytes.len().min(want - dst); + out[dst..dst + n].copy_from_slice(&bytes[..n]); + } + } + + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filer_pb::FileId; + + #[test] + fn fileid_format_strips_leading_key_zeros() { + // vid 7, key 0x1234, cookie 0xaabbccdd + let fid = FileId { + volume_id: 7, + file_key: 0x1234, + cookie: 0xaabb_ccdd, + }; + // key 8 bytes BE = 00 00 00 00 00 00 12 34 -> strip 6 leading zeros -> 12 34 + // cookie = aa bb cc dd => "1234aabbccdd" + assert_eq!(fid_to_string(&fid), "7,1234aabbccdd"); + } + + #[test] + fn fileid_zero_key_keeps_cookie() { + let fid = FileId { + volume_id: 3, + file_key: 0, + cookie: 0x0000_00ff, + }; + assert_eq!(fid_to_string(&fid), "3,000000ff"); + } + + use async_trait::async_trait; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Counts how many actual transport reads (network fetches) happen; returns + /// the fileId bytes so callers can check correctness. + struct MockTransport { + reads: AtomicUsize, + } + impl MockTransport { + fn new() -> Self { + Self { + reads: AtomicUsize::new(0), + } + } + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) + } + } + #[async_trait] + impl VolumeTransport for MockTransport { + async fn read( + &self, + _locations: &[String], + file_id: &str, + _range: Option<(u64, u64)>, + ) -> Result> { + self.reads.fetch_add(1, Ordering::SeqCst); + Ok(file_id.as_bytes().to_vec()) + } + async fn write( + &self, + _location: &str, + _file_id: &str, + _auth: &str, + _filename: &str, + _data: bytes::Bytes, + ) -> Result<()> { + Ok(()) + } + fn name(&self) -> &'static str { + "mock" + } + } + + async fn run_pending_prefetch(mock: &MockTransport, want: usize) { + for _ in 0..10_000 { + if mock.reads() >= want { + return; + } + tokio::task::yield_now().await; + } + panic!("prefetch did not complete (reads={})", mock.reads()); + } + + /// Always fails reads — to exercise the fetch error path. + struct FailingTransport; + #[async_trait] + impl VolumeTransport for FailingTransport { + async fn read( + &self, + _locations: &[String], + _file_id: &str, + _range: Option<(u64, u64)>, + ) -> Result> { + bail!("simulated volume read failure") + } + async fn write( + &self, + _location: &str, + _file_id: &str, + _auth: &str, + _filename: &str, + _data: bytes::Bytes, + ) -> Result<()> { + Ok(()) + } + fn name(&self) -> &'static str { + "failing" + } + } + + #[tokio::test] + async fn fetch_whole_caches_by_fileid() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + let urls = vec!["h:1".to_string()]; + let a = vr.fetch_and_cache_whole(&urls, "1,aa").await.unwrap(); + let b = vr.fetch_and_cache_whole(&urls, "1,aa").await.unwrap(); + assert_eq!(&*a, b"1,aa"); // correct bytes + assert_eq!(&*a, &*b); + assert_eq!(mock.reads(), 1, "the second read must hit the cache"); + } + + #[test] + fn gate_guard_only_removes_its_own_gate() { + // A waiter's guard must not evict a newer gate installed for the same + // fileId by a later request (which would defeat that request's + // single-flight). Pointer-equality is what protects it. + let inflight: Arc>>>> = + Arc::new(Mutex::new(HashMap::new())); + let mine = Arc::new(tokio::sync::Mutex::new(())); + inflight.lock().unwrap().insert("1,aa".into(), mine.clone()); + let guard = GateGuard { + inflight: inflight.clone(), + file_id: "1,aa".into(), + gate: mine.clone(), + }; + // A later request replaces the gate for the same fileId. + let newer = Arc::new(tokio::sync::Mutex::new(())); + inflight + .lock() + .unwrap() + .insert("1,aa".into(), newer.clone()); + drop(guard); // must leave `newer` in place + let current = inflight.lock().unwrap().get("1,aa").cloned(); + assert!( + current.is_some_and(|g| Arc::ptr_eq(&g, &newer)), + "guard evicted a newer gate belonging to another request" + ); + } + + #[tokio::test] + async fn single_flight_gate_is_released_on_success() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + vr.fetch_and_cache_whole(&["h:1".to_string()], "1,aa") + .await + .unwrap(); + assert!( + vr.inflight.lock().unwrap().is_empty(), + "the single-flight gate must be dropped after a completed fetch" + ); + } + + #[tokio::test] + async fn single_flight_gate_is_released_on_error() { + // A failed fetch must NOT leave its gate behind: otherwise `inflight` + // would grow by one entry per distinct fileId that ever failed to fetch. + let vr = VolumeReader::with_transport(Arc::new(FailingTransport)); + for i in 0..100 { + let fid = format!("1,{i:x}"); + assert!( + vr.fetch_and_cache_whole(&["h:1".to_string()], &fid) + .await + .is_err() + ); + } + assert!( + vr.inflight.lock().unwrap().is_empty(), + "failed fetches must not leak single-flight gates" + ); + } + + #[tokio::test] + async fn failed_fetch_invalidates_the_volume_locations() { + // Without this the mount would keep retrying servers a volume has moved + // off, with no way back short of a remount. + let vr = VolumeReader::with_transport(Arc::new(FailingTransport)); + let urls = vec!["moved-from:8080".to_string()]; + vr.location_cache.learn(7, urls.clone()); + vr.location_cache.learn(8, urls.clone()); + + assert!(vr.fetch_and_cache_whole(&urls, "7,aa").await.is_err()); + + assert_eq!( + vr.cached_locations(7), + None, + "a failed fetch must force the next read to look the volume up again" + ); + assert!( + vr.cached_locations(8).is_some(), + "only the volume that failed should be invalidated" + ); + } + + #[tokio::test] + async fn failed_ranged_fetch_invalidates_the_volume_locations() { + let vr = VolumeReader::with_transport(Arc::new(FailingTransport)); + let urls = vec!["moved-from:8080".to_string()]; + vr.location_cache.learn(7, urls.clone()); + + assert!(vr.fetch_ranged(&urls, "7,aa", 0, 4, false).await.is_err()); + + assert_eq!(vr.cached_locations(7), None); + } + + #[tokio::test] + async fn failed_fetch_keeps_locations_a_newer_lookup_learned() { + // A read can fail long after another lookup has learned where the + // volume moved to; dropping that fresh entry would send every reader + // back to the filer for nothing. + let vr = VolumeReader::with_transport(Arc::new(FailingTransport)); + let stale = vec!["moved-from:8080".to_string()]; + vr.location_cache + .learn(7, vec!["moved-to:8080".to_string()]); + + assert!(vr.fetch_and_cache_whole(&stale, "7,aa").await.is_err()); + + assert_eq!( + vr.cached_locations(7), + Some(vec!["moved-to:8080".to_string()]), + "a stale read's failure must not invalidate freshly learned locations" + ); + } + + #[tokio::test] + async fn successful_fetch_keeps_the_cached_locations() { + let vr = VolumeReader::with_transport(Arc::new(MockTransport::new())); + let urls = vec!["h:1".to_string()]; + vr.location_cache.learn(7, urls.clone()); + + vr.fetch_and_cache_whole(&urls, "7,aa").await.unwrap(); + + assert_eq!(vr.cached_locations(7), Some(urls)); + } + + #[tokio::test] + async fn reset_locations_keeps_serving_until_relearned() { + let vr = VolumeReader::with_transport(Arc::new(MockTransport::new())); + let urls = vec!["h:1".to_string()]; + vr.location_cache.learn(7, urls.clone()); + + vr.reset_locations(); + + assert_eq!( + vr.cached_locations(7), + Some(urls), + "a reset must not strand reads that are already in flight" + ); + } + + #[tokio::test] + async fn prefetch_fills_cache_then_serves_hit() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + let urls = vec!["h:1".to_string()]; + vr.prefetch(vec![("1,bb".to_string(), urls.clone())]); + run_pending_prefetch(&mock, 1).await; + // a real fetch is now served from the prefetched cache — no extra read. + let got = vr.fetch_and_cache_whole(&urls, "1,bb").await.unwrap(); + assert_eq!(&*got, b"1,bb"); + assert_eq!( + mock.reads(), + 1, + "fetch_whole should hit the prefetched chunk" + ); + } + + #[tokio::test] + async fn prefetch_skips_already_cached() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + let urls = vec!["h:1".to_string()]; + let _ = vr.fetch_and_cache_whole(&urls, "1,cc").await.unwrap(); // reads=1, now cached + vr.prefetch(vec![("1,cc".to_string(), urls.clone())]); // cached -> must skip + for _ in 0..200 { + tokio::task::yield_now().await; + } + assert_eq!(mock.reads(), 1, "prefetch must not refetch a cached chunk"); + } + + const MB: u64 = 1 << 20; + + fn reader() -> VolumeReader { + VolumeReader::with_transport(Arc::new(MockTransport::new())) + } + + fn chunk(file_id: &str, offset: i64, size: u64, modified_ts_ns: i64) -> FileChunk { + FileChunk { + file_id: file_id.to_string(), + offset, + size, + modified_ts_ns, + ..Default::default() + } + } + + #[test] + fn resolve_ordered_non_overlapping_chunks_without_shadow_work() { + let chunks = vec![chunk("1,aa", 0, MB, 3), chunk("1,bb", 2 * MB as i64, MB, 1)]; + let visibles = resolve_visibles(&chunks); + + assert_eq!(visibles.len(), 2); + assert_eq!( + ( + visibles[0].start, + visibles[0].stop, + visibles[0].file_id.as_str() + ), + (0, MB as i64, "1,aa") + ); + assert_eq!( + ( + visibles[1].start, + visibles[1].stop, + visibles[1].file_id.as_str() + ), + (2 * MB as i64, 3 * MB as i64, "1,bb") + ); + } + + #[test] + fn resolve_overlapping_chunks_still_uses_latest_write_wins() { + let chunks = vec![chunk("1,old", 0, 8, 1), chunk("1,new", 4, 4, 2)]; + let visibles = resolve_visibles(&chunks); + + assert_eq!(visibles.len(), 2); + assert_eq!( + ( + visibles[0].start, + visibles[0].stop, + visibles[0].file_id.as_str() + ), + (0, 4, "1,old") + ); + assert_eq!( + ( + visibles[1].start, + visibles[1].stop, + visibles[1].file_id.as_str() + ), + (4, 8, "1,new") + ); + } + + #[test] + fn gate_sequential_ramps_and_resists_outlier() { + let vr = reader(); + // a stream from offset 0 is sequential and ramps the counter to the cap + assert!(vr.note_sequential("f", 0, MB)); // near(0 vs frontier 0) -> +1 + assert!(vr.note_sequential("f", MB, 2 * MB)); // +2 + assert!(vr.note_sequential("f", 2 * MB, 3 * MB)); // +3 (capped) + // a single far outlier does NOT flip to random (hysteresis): +3 -> +2 + assert!(vr.note_sequential("f", 900 * MB, 901 * MB)); + } + + #[test] + fn gate_far_first_read_is_random() { + let vr = reader(); + // first read far into the file (vs frontier 0) -> random, no prefetch + assert!(!vr.note_sequential("f", 500 * MB, 501 * MB)); + } + + #[test] + fn gate_tolerance_absorbs_reorder() { + let vr = reader(); + assert!(vr.note_sequential("f", 0, MB)); // frontier -> 1MB, +1 + // a read jumping ahead by 5 MB (< tolerance) — reordered readahead — is near + assert!(vr.note_sequential("f", 6 * MB, 7 * MB)); // |6-1|=5MB <= 8MB, frontier 7MB, +2 + // a read behind/overlapping the frontier, within tolerance, is also near + assert!(vr.note_sequential("f", 3 * MB, 4 * MB)); // |3-7|=4MB <= 8MB, +3 + } + + #[test] + fn gate_random_recovery_needs_hysteresis() { + let vr = reader(); + assert!(!vr.note_sequential("f", 100 * MB, 101 * MB)); // far -> -1 (random) + let _ = vr.note_sequential("f", 300 * MB, 301 * MB); // far -> -2 + let _ = vr.note_sequential("f", 500 * MB, 501 * MB); // far -> -3 (capped) + // a single near read does NOT immediately flip back to sequential: -3 -> -2 + assert!(!vr.note_sequential("f", 501 * MB, 502 * MB)); + } + + #[tokio::test] + async fn random_fetch_is_exact_and_uncached() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + let urls = vec!["h:1".to_string()]; + let _ = vr.fetch_ranged(&urls, "1,dd", 0, 16, false).await.unwrap(); + let _ = vr.fetch_ranged(&urls, "1,dd", 0, 16, false).await.unwrap(); + assert_eq!(mock.reads(), 2, "random reads must not be cached"); + } + + #[tokio::test] + async fn sequential_fetch_caches_whole() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + let urls = vec!["h:1".to_string()]; + let a = vr.fetch_ranged(&urls, "1,ee", 0, 2, true).await.unwrap(); + assert_eq!(a, b"1,"); // sliced from the cached whole needle + // a later sub-range of the same chunk hits the cache even in random mode. + let b2 = vr.fetch_ranged(&urls, "1,ee", 2, 2, false).await.unwrap(); + assert_eq!(b2, b"ee"); + assert_eq!(mock.reads(), 1, "whole-chunk cache serves all sub-ranges"); + } + + /// Serves byte ranges of named needles. + struct BytesTransport { + needles: HashMap>, + reads: AtomicUsize, + whole_reads: AtomicUsize, + } + + #[async_trait] + impl VolumeTransport for BytesTransport { + async fn read( + &self, + _locations: &[String], + file_id: &str, + range: Option<(u64, u64)>, + ) -> Result> { + self.reads.fetch_add(1, Ordering::SeqCst); + let needle = self.needles.get(file_id).context("no such needle")?; + Ok(match range { + Some((off, len)) => needle[off as usize..(off + len) as usize].to_vec(), + None => { + self.whole_reads.fetch_add(1, Ordering::SeqCst); + needle.clone() + } + }) + } + async fn write( + &self, + _location: &str, + _file_id: &str, + _auth: &str, + _filename: &str, + _data: bytes::Bytes, + ) -> Result<()> { + Ok(()) + } + fn name(&self) -> &'static str { + "bytes" + } + } + + /// Encrypted chunks are stored XORed with their stream position; chunks + /// with a cipher key can only be undone whole. + struct XorDecoder; + + fn xor(data: &mut [u8], position: u64) { + for (i, b) in data.iter_mut().enumerate() { + *b ^= (position as usize + i) as u8 ^ 0x5a; + } + } + + impl ChunkDecoder for XorDecoder { + fn needs_whole_chunk(&self, chunk: &FileChunk) -> bool { + !chunk.cipher_key.is_empty() + } + fn decode(&self, chunk: &FileChunk, offset: u64, mut data: Vec) -> Result> { + if encoded(chunk) { + xor(&mut data, offset); + } + Ok(data) + } + } + + /// A 32-byte file: a plain chunk, a manifest holding a plain chunk, an + /// SSE chunk and a filer-encrypted chunk, 8 bytes each. + fn encrypted_file() -> (Entry, Vec, Arc, VolumeReader) { + let plain: Vec = (0..32u8).map(|i| i.wrapping_mul(37)).collect(); + let mut sse = plain[16..24].to_vec(); + xor(&mut sse, 0); + let mut gcm = plain[24..32].to_vec(); + xor(&mut gcm, 0); + let inner = FileChunkManifest { + chunks: vec![chunk("1,bb", 8, 8, 1)], + }; + let needles = HashMap::from([ + ("1,aa".to_string(), plain[..8].to_vec()), + ("1,mm".to_string(), inner.encode_to_vec()), + ("1,bb".to_string(), plain[8..16].to_vec()), + ("1,ee".to_string(), sse), + ("1,gg".to_string(), gcm), + ]); + let transport = Arc::new(BytesTransport { + needles, + reads: AtomicUsize::new(0), + whole_reads: AtomicUsize::new(0), + }); + let volumes = VolumeReader::with_transport(transport.clone()); + volumes.location_cache.learn(1, vec!["h:1".into()]); + let mut manifest = chunk("1,mm", 8, 8, 1); + manifest.is_chunk_manifest = true; + let mut sse_chunk = chunk("1,ee", 16, 8, 1); + sse_chunk.sse_type = 3; + let mut gcm_chunk = chunk("1,gg", 24, 8, 1); + gcm_chunk.cipher_key = vec![7; 32]; + let entry = Entry { + attributes: Some(crate::filer_pb::FuseAttributes { + file_size: 32, + ..Default::default() + }), + chunks: vec![chunk("1,aa", 0, 8, 1), manifest, sse_chunk, gcm_chunk], + ..Default::default() + }; + (entry, plain, transport, volumes) + } + + #[tokio::test] + async fn manifests_expand_and_the_decoder_sees_every_chunk() { + let (entry, plain, transport, volumes) = encrypted_file(); + let mut client = FilerClient::connect_lazy("127.0.0.1:1").unwrap(); + let options = ReadOptions { + exact: true, + decoder: Some(Arc::new(XorDecoder)), + }; + let got = read_file_with(&mut client, &volumes, &entry, 0, 32, &options) + .await + .unwrap(); + assert_eq!(got, plain); + + // An unaligned read into the SSE chunk is fetched as a range and + // decoded at its offset; the filer-encrypted chunk is fetched whole. + let resolved = resolve_chunks(&mut client, &volumes, &entry.chunks, &options) + .await + .unwrap(); + assert_eq!(resolved.end(), 32); + let before = transport.reads.load(Ordering::SeqCst); + let whole_before = transport.whole_reads.load(Ordering::SeqCst); + let got = read_resolved(&mut client, &volumes, &resolved, &entry, 19, 10, &options) + .await + .unwrap(); + assert_eq!(got, &plain[19..29]); + assert_eq!(transport.reads.load(Ordering::SeqCst) - before, 2); + assert_eq!( + transport.whole_reads.load(Ordering::SeqCst) - whole_before, + 1 + ); + assert!( + volumes.chunks.lock().unwrap().len_bytes() == 0, + "exact reads cache nothing" + ); + + let tail = read_resolved( + &mut client, + &volumes, + &resolved, + &entry, + 19, + u64::MAX, + &options, + ) + .await + .unwrap(); + assert_eq!(tail, &plain[19..], "a size past u64::MAX reads to EOF"); + } + + #[tokio::test] + async fn encrypted_chunks_need_a_decoder() { + let (entry, plain, _, volumes) = encrypted_file(); + let mut client = FilerClient::connect_lazy("127.0.0.1:1").unwrap(); + let err = read_file(&mut client, &volumes, &entry, 16, 8) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("needs a decoder"), "{err:#}"); + let got = read_file(&mut client, &volumes, &entry, 0, 16) + .await + .unwrap(); + assert_eq!( + got, + &plain[..16], + "plain chunks and manifests read without one" + ); + + let mut sse_manifest = entry.clone(); + sse_manifest.chunks[1].sse_type = 3; + let err = read_file(&mut client, &volumes, &sse_manifest, 0, 8) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("needs a decoder"), "{err:#}"); + } + + #[tokio::test] + async fn sequential_fetch_into_copies_from_cached_whole() { + let mock = Arc::new(MockTransport::new()); + let vr = VolumeReader::with_transport(mock.clone()); + let urls = vec!["h:1".to_string()]; + + let mut first = [0u8; 2]; + let n = vr + .fetch_ranged_into(&urls, "1,ee", 0, 2, true, &mut first) + .await + .unwrap(); + assert_eq!(n, 2); + assert_eq!(&first, b"1,"); + + let mut second = [0u8; 2]; + let n = vr + .fetch_ranged_into(&urls, "1,ee", 2, 2, false, &mut second) + .await + .unwrap(); + assert_eq!(n, 2); + assert_eq!(&second, b"ee"); + assert_eq!( + mock.reads(), + 1, + "second copy should use the cached whole chunk" + ); + } +} diff --git a/seaweed-worker/crates/sw-client/src/transport.rs b/seaweed-worker/crates/sw-client/src/transport.rs new file mode 100644 index 0000000000..d3f85e4b7b --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/transport.rs @@ -0,0 +1,633 @@ +//! Volume data-plane transport: moves needle bytes to/from volume servers. +//! +//! Everything *above* this — resolving a file's chunks, looking up a volume's +//! locations, assigning a volume for a write — stays in the filer client. A +//! transport only transfers bytes for an already-resolved `fileId` at an +//! already-known location, so it is the single seam where the wire technology +//! lives: HTTP today ([`HttpTransport`]), RDMA (SoftRoCE / real NICs) next. +//! +//! The trait is deliberately narrow (read a range, write a needle) so an RDMA +//! implementation — registered memory regions, queue pairs, one-sided READ — +//! drops in without touching `read.rs` / `write.rs` or the kernel daemon. + +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use bytes::Bytes; +use reqwest::header::{AUTHORIZATION, RANGE}; +use std::time::Duration; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct VolumeWriteReceipt { + /// SHA-256 computed by the volume over the bytes it committed. `None` + /// means the transport/server pair does not advertise that proof. + pub committed_sha256: Option, + /// Exact volume endpoint that accepted this commit. Fenced capacity writes + /// persist it with their ownership receipt so GC can retire the original + /// writer even after the volume topology changes. + pub committed_location: Option, +} + +/// Transfers needle bytes for a resolved `fileId` + location. Implementations +/// must be cheap to clone/share (wrapped in `Arc`) and safe for concurrent use. +#[async_trait] +pub trait VolumeTransport: Send + Sync { + /// Read `file_id` from any one of `locations` (`host:port`), optionally the + /// logical byte range `(offset, len)`. Returns the (server-decompressed) + /// needle bytes. Implementations should try locations in order until one + /// succeeds. + async fn read( + &self, + locations: &[String], + file_id: &str, + range: Option<(u64, u64)>, + ) -> Result>; + + /// Read with a short-lived exact-FID capability. Transports which cannot + /// carry read authorization fail closed instead of silently dropping it. + async fn read_with_auth( + &self, + locations: &[String], + file_id: &str, + range: Option<(u64, u64)>, + auth: &str, + ) -> Result> { + if !auth.is_empty() { + bail!( + "{} transport cannot carry an exact-FID read capability", + self.name() + ); + } + self.read(locations, file_id, range).await + } + + /// Upload `data` as needle `file_id` to `location` (`host:port`). `filename` + /// names the needle (volume servers use it for content-type/compression + /// heuristics); `auth` is the per-fileId write JWT (`""` if none). + async fn write( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + ) -> Result<()>; + + /// Upload and return any proof produced by the volume commit. The default + /// keeps existing transports source-compatible without claiming a digest. + async fn write_with_receipt( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + ) -> Result { + self.write(location, file_id, auth, filename, data).await?; + Ok(VolumeWriteReceipt::default()) + } + + /// [`Self::write_with_receipt`] with the volume told to sync the needle to + /// disk before it answers, when `fsync` is set. A transport that cannot ask + /// for that fails closed rather than acknowledge a write still in memory. + async fn write_synced( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + fsync: bool, + ) -> Result { + if fsync { + bail!("{} transport cannot request fsync", self.name()); + } + self.write_with_receipt(location, file_id, auth, filename, data) + .await + } + + /// Short label for logs/diagnostics (e.g. `"http"`, `"rdma"`). + fn name(&self) -> &'static str; +} + +/// Marker a [`VolumeTransport::write`] attaches when the write may have already +/// been committed but its outcome is unknown, so retrying or falling back could +/// duplicate data. Callers should fence the write instead of retrying; detect it +/// with `err.chain().any(|c| c.is::())`. +#[derive(Debug)] +pub struct CommitAmbiguous; + +impl std::fmt::Display for CommitAmbiguous { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "write commit status unknown; no safe fallback") + } +} + +impl std::error::Error for CommitAmbiguous {} + +/// Marker a [`VolumeTransport::write`] attaches when the Volume refused the +/// write before any of it was committed, for example a required RDMA write to +/// a Volume that was sealed after the assignment. The needle does not exist, +/// so the caller may write it again under a fresh assignment; detect it with +/// [`is_not_written`]. +#[derive(Debug)] +pub struct NotWritten; + +impl std::fmt::Display for NotWritten { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "write refused before commit; nothing was written") + } +} + +impl std::error::Error for NotWritten {} + +/// Whether `error` carries [`NotWritten`] anywhere in its context chain. +pub fn is_not_written(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +/// Connect bound for every volume request: a server that never answers a SYN +/// (silently partitioned) must not hold a read or a write. +pub const VOLUME_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +/// Bound on one GET from one location, connect to end of body. Two locations +/// fit inside a caller's 30 s op budget, so a silent first location fails +/// over to the next instead of holding the read (chaos C3). +pub const VOLUME_READ_TIMEOUT: Duration = Duration::from_secs(10); +/// Bound on one upload. Longer than the volume server's own 10 s bound on its +/// replica forwards, so a dead replica comes back as the server's error +/// first; still inside a caller's 30 s op budget. +pub const VOLUME_WRITE_TIMEOUT: Duration = Duration::from_secs(20); + +/// TLS for volume servers, as `https.client` in security.toml configures it +/// for weed processes: enabled, volume servers are dialed over HTTPS, their +/// certificates checked against `ca` (the system roots without one), and +/// this client presents `cert`/`key` when set. +#[derive(Clone, Default)] +pub struct HttpsTls { + pub ca_pem: Option>, + pub cert_pem: Option>, + pub key_pem: Option>, + /// `https.client.insecure_skip_verify` + pub insecure_skip_verify: bool, +} + +impl std::fmt::Debug for HttpsTls { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpsTls") + .field("ca", &self.ca_pem.is_some()) + .field("identity", &self.cert_pem.is_some()) + .field("insecure_skip_verify", &self.insecure_skip_verify) + .finish() + } +} + +/// Both tonic and reqwest link rustls; with more than one crypto provider +/// compiled in, rustls needs one picked before TLS is configured. +pub fn install_crypto_provider() { + let _ = rustls::crypto::ring::default_provider().install_default(); +} + +/// Default transport: the SeaweedFS volume HTTP API — `GET /` with an +/// optional `Range` header, and a multipart `POST` to upload a needle. +pub struct HttpTransport { + http: reqwest::Client, + scheme: &'static str, + read_timeout: Duration, + write_timeout: Duration, +} + +impl Default for HttpTransport { + fn default() -> Self { + Self::new() + } +} + +impl HttpTransport { + pub fn new() -> Self { + Self::with_timeouts(VOLUME_READ_TIMEOUT, VOLUME_WRITE_TIMEOUT) + } + + fn with_timeouts(read_timeout: Duration, write_timeout: Duration) -> Self { + Self { + http: reqwest::Client::builder() + .connect_timeout(VOLUME_CONNECT_TIMEOUT) + .build() + .expect("volume HTTP client"), + scheme: "http", + read_timeout, + write_timeout, + } + } + + /// A transport dialing volume servers over HTTPS. + pub fn with_tls(tls: &HttpsTls) -> Result { + install_crypto_provider(); + let mut builder = reqwest::Client::builder() + .connect_timeout(VOLUME_CONNECT_TIMEOUT) + .use_rustls_tls() + .danger_accept_invalid_certs(tls.insecure_skip_verify); + if let Some(ca) = &tls.ca_pem { + builder = builder.tls_built_in_root_certs(false).add_root_certificate( + reqwest::Certificate::from_pem(ca).context("https.client.ca")?, + ); + } + match (&tls.cert_pem, &tls.key_pem) { + (Some(cert), Some(key)) => { + let mut pem = cert.clone(); + if !pem.ends_with(b"\n") { + pem.push(b'\n'); + } + pem.extend_from_slice(key); + builder = builder.identity( + reqwest::Identity::from_pem(&pem).context("https.client cert and key")?, + ); + } + (None, None) => {} + _ => bail!("https.client needs both cert and key, or neither"), + } + Ok(Self { + http: builder.build().context("volume HTTPS client")?, + scheme: "https", + read_timeout: VOLUME_READ_TIMEOUT, + write_timeout: VOLUME_WRITE_TIMEOUT, + }) + } +} + +#[async_trait] +impl VolumeTransport for HttpTransport { + async fn read( + &self, + locations: &[String], + file_id: &str, + range: Option<(u64, u64)>, + ) -> Result> { + self.read_with_auth(locations, file_id, range, "").await + } + + async fn read_with_auth( + &self, + locations: &[String], + file_id: &str, + range: Option<(u64, u64)>, + auth: &str, + ) -> Result> { + let range_header = match range { + Some((_, 0)) => return Ok(Vec::new()), + Some((start, len)) => { + let end = start + .checked_add(len - 1) + .with_context(|| format!("range {start}+{len} of {file_id}"))?; + Some(format!("bytes={start}-{end}")) + } + None => None, + }; + let mut last_err = None; + for url in locations { + let full = format!("{}://{url}/{file_id}", self.scheme); + let mut req = self.http.get(&full).timeout(self.read_timeout); + if let Some(range) = &range_header { + req = req.header(RANGE, range); + } + if !auth.is_empty() { + req = req.header(AUTHORIZATION, format!("BEARER {auth}")); + } + match req.send().await { + Ok(resp) if resp.status().is_success() => { + // A 200 to a ranged GET is the whole needle. + let whole = resp.status() != reqwest::StatusCode::PARTIAL_CONTENT; + match resp.bytes().await { + Ok(b) => { + return Ok(match range { + Some((start, len)) if whole => { + let start = start.min(b.len() as u64) as usize; + let end = start + (len.min((b.len() - start) as u64) as usize); + b[start..end].to_vec() + } + _ => b.to_vec(), + }); + } + Err(e) => last_err = Some(e.into()), + } + } + Ok(resp) => last_err = Some(anyhow!("GET {full} -> HTTP {}", resp.status())), + Err(e) => last_err = Some(e.into()), + } + } + Err(last_err.unwrap_or_else(|| anyhow!("no volume URL for {file_id}"))) + } + + async fn write( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + ) -> Result<()> { + self.upload(location, file_id, auth, filename, data, false) + .await + } + + async fn write_synced( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + fsync: bool, + ) -> Result { + self.upload(location, file_id, auth, filename, data, fsync) + .await?; + Ok(VolumeWriteReceipt::default()) + } + + fn name(&self) -> &'static str { + "http" + } +} + +impl HttpTransport { + async fn upload( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + fsync: bool, + ) -> Result<()> { + let mut url = format!("{}://{location}/{file_id}", self.scheme); + if fsync { + url.push_str("?fsync=true"); + } + let size = data.len() as u64; + let part = reqwest::multipart::Part::stream_with_length(data, size) + .file_name(filename.to_string()); + let form = reqwest::multipart::Form::new().part("file", part); + let mut req = self + .http + .post(&url) + .timeout(self.write_timeout) + .multipart(form); + if !auth.is_empty() { + req = req.header(AUTHORIZATION, format!("BEARER {auth}")); + } + let resp = match req.send().await { + Ok(resp) => resp, + // Timed out after the request went out: the volume may have + // committed it, so callers must fence instead of retrying. + Err(e) if e.is_timeout() && !e.is_connect() => { + return Err(e) + .with_context(|| format!("upload to {url}")) + .context(CommitAmbiguous); + } + Err(e) => return Err(e).with_context(|| format!("upload to {url}")), + }; + if !resp.status().is_success() { + let error = anyhow!("upload {url} -> HTTP {}", resp.status()); + // Only an explicit pre-append refusal permits reassignment. + if resp.status() == reqwest::StatusCode::CONFLICT + && resp + .headers() + .get("x-seaweed-write-outcome") + .is_some_and(|value| value == "refused-before-append") + { + return Err(error.context(NotWritten)); + } + return Err(error); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + #[test] + fn not_written_is_found_under_later_contexts_and_only_there() { + use anyhow::Context; + let refused: anyhow::Result<()> = Err(anyhow::anyhow!("prepare rejected")) + .context(NotWritten) + .context("required RDMA write failed in PrepareRejected"); + let wrapped = refused + .context("upload 38,03e1 to 10.0.2.3:19601") + .context("loader write /lmcache/k") + .unwrap_err(); + assert!(is_not_written(&wrapped)); + + let ambiguous: anyhow::Result<()> = + Err(anyhow::anyhow!("commit rpc timed out")).context(CommitAmbiguous); + assert!(!is_not_written(&ambiguous.context("upload").unwrap_err())); + } + + #[tokio::test] + async fn authenticated_read_sends_exact_fid_bearer_token() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let length = socket.read(&mut request).await.unwrap(); + let request = String::from_utf8_lossy(&request[..length]).to_ascii_lowercase(); + assert!(request.starts_with("get /7,abc http/1.1\r\n")); + assert!(request.contains("\r\nauthorization: bearer signed-token\r\n")); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nabc") + .await + .unwrap(); + }); + + let bytes = HttpTransport::new() + .read_with_auth(&[address.to_string()], "7,abc", None, "signed-token") + .await + .unwrap(); + assert_eq!(bytes, b"abc"); + server.await.unwrap(); + } + + /// A server that ignores Range and answers 200 with the whole needle + /// yields only the bytes asked for. + #[tokio::test] + async fn a_whole_needle_answer_to_a_ranged_read_is_sliced() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let server = tokio::spawn(async move { + for _ in 0..3 { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let _ = socket.read(&mut request).await.unwrap(); + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n0123456789", + ) + .await + .unwrap(); + } + }); + let http = HttpTransport::new(); + let at = std::slice::from_ref(&address); + assert_eq!(http.read(at, "7,abc", Some((3, 4))).await.unwrap(), b"3456"); + assert_eq!(http.read(at, "7,abc", Some((8, 4))).await.unwrap(), b"89"); + assert!( + http.read(at, "7,abc", Some((12, 4))) + .await + .unwrap() + .is_empty() + ); + server.await.unwrap(); + } + + /// An empty range needs no request, and one past the end of the address + /// space sends none. + #[tokio::test] + async fn empty_and_overflowing_ranges_send_no_request() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let closed = listener.local_addr().unwrap().to_string(); + drop(listener); + let http = HttpTransport::new(); + let read = http + .read(std::slice::from_ref(&closed), "7,abc", Some((5, 0))) + .await + .unwrap(); + assert!(read.is_empty()); + let error = http + .read(&[closed], "7,abc", Some((u64::MAX, 2))) + .await + .expect_err("the range cannot be expressed"); + assert!(format!("{error:#}").contains("range"), "{error:#}"); + } + + /// A server that accepts the connection and never answers: a silently + /// partitioned volume server after the connect (chaos C3). + async fn silent_server() -> (String, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let held = tokio::spawn(async move { + let mut sockets = Vec::new(); + while let Ok((socket, _)) = listener.accept().await { + sockets.push(socket); + } + }); + (address, held) + } + + /// Chaos C3: a GET with the silent copy listed first must reach the + /// healthy copy within its per-location bound, not wait on the first. + #[tokio::test] + async fn read_fails_over_from_a_silent_location() { + let (silent, held) = silent_server().await; + let healthy = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let healthy_address = healthy.local_addr().unwrap().to_string(); + let server = tokio::spawn(async move { + let (mut socket, _) = healthy.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let _ = socket.read(&mut request).await.unwrap(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nabc") + .await + .unwrap(); + }); + let bound = Duration::from_millis(300); + let started = std::time::Instant::now(); + let bytes = tokio::time::timeout( + Duration::from_secs(10), + HttpTransport::with_timeouts(bound, bound).read( + &[silent, healthy_address], + "7,abc", + None, + ), + ) + .await + .expect("a silent first location must not hold the read") + .unwrap(); + assert_eq!(bytes, b"abc"); + assert!(started.elapsed() >= bound && started.elapsed() < bound * 10); + server.await.unwrap(); + held.abort(); + } + + /// A write to a silent server fails at its bound instead of waiting. + #[tokio::test] + async fn write_to_a_silent_server_fails_at_its_bound() { + let (silent, held) = silent_server().await; + let bound = Duration::from_millis(300); + let started = std::time::Instant::now(); + let written = tokio::time::timeout( + Duration::from_secs(10), + HttpTransport::with_timeouts(bound, bound).write( + &silent, + "7,abc", + "", + "k", + Bytes::from(vec![0u8; 1 << 20]), + ), + ) + .await + .expect("a silent server must not hold the write"); + // The request went out, so the commit is unknown: fence, never + // reassign. Detected through the context, as sw-kd does (#918). + let error = written.expect_err("a silent server cannot ack the write"); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + assert!(!is_not_written(&error)); + assert!(started.elapsed() >= bound && started.elapsed() < bound * 10); + held.abort(); + } + + /// An assignment that wants the needle on disk asks the volume for it. + #[tokio::test] + async fn synced_write_asks_the_volume_to_fsync() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let server = tokio::spawn(async move { + let mut lines = Vec::new(); + for _ in 0..2 { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let length = socket.read(&mut request).await.unwrap(); + let request = String::from_utf8_lossy(&request[..length]).to_string(); + lines.push(request.lines().next().unwrap_or_default().to_string()); + socket + .write_all( + b"HTTP/1.1 201 Created\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + } + lines + }); + let http = HttpTransport::new(); + for fsync in [true, false] { + http.write_synced(&address, "7,abc", "", "k", Bytes::from_static(b"kv"), fsync) + .await + .unwrap(); + } + assert_eq!( + server.await.unwrap(), + ["POST /7,abc?fsync=true HTTP/1.1", "POST /7,abc HTTP/1.1"] + ); + } + + /// A write that never reached the server (connection refused) is a plain + /// error, not an ambiguous commit. + #[tokio::test] + async fn write_that_never_connected_is_not_ambiguous() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let closed = listener.local_addr().unwrap().to_string(); + drop(listener); + let error = HttpTransport::new() + .write(&closed, "7,abc", "", "k", Bytes::from_static(b"kv")) + .await + .expect_err("nothing listens there"); + assert!( + error.downcast_ref::().is_none(), + "{error:#}" + ); + } +} diff --git a/seaweed-worker/crates/sw-client/src/wbuf.rs b/seaweed-worker/crates/sw-client/src/wbuf.rs new file mode 100644 index 0000000000..0fd252e500 --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/wbuf.rs @@ -0,0 +1,348 @@ +//! Streaming write buffer: stages dirty file bytes per chunk so a flush can +//! upload completed chunks incrementally and free them, keeping resident memory +//! bounded (a few chunks) regardless of file size — instead of holding the whole +//! file in RAM. +//! +//! Modelled on the SeaweedFS Go mount page writer (`weed/mount/page_writer`): +//! dirty bytes are keyed by chunk index, a fully-written chunk is "sealed" and +//! uploaded as an overlay `FileChunk` (newest wins by `modified_ts_ns` at read +//! time, see [`crate::read::resolve_visibles`]), and reads merge committed +//! chunks with the still-staged bytes. +//! +//! This type is pure (no I/O): the daemon drives the actual uploads. It tracks +//! *which* bytes are dirty (per-chunk written intervals) so that a partially +//! written chunk only ever uploads the bytes the caller actually wrote — never +//! zero-filling a gap, which would clobber underlying committed data. + +use std::collections::BTreeMap; + +/// Dirty bytes for one chunk index. `data` is dense from offset 0; only the +/// ranges listed in `intervals` (chunk-relative, merged, sorted, non-adjacent) +/// hold caller-written bytes. +struct StagedChunk { + data: Vec, + intervals: Vec<(usize, usize)>, +} + +impl StagedChunk { + fn new() -> Self { + Self { + data: Vec::new(), + intervals: Vec::new(), + } + } + + /// Mark `[lo, hi)` written, coalescing with touching/overlapping intervals. + fn mark(&mut self, lo: usize, hi: usize) { + let mut merged = (lo, hi); + let mut out: Vec<(usize, usize)> = Vec::with_capacity(self.intervals.len() + 1); + for &(s, e) in &self.intervals { + if e < merged.0 || s > merged.1 { + out.push((s, e)); // disjoint (note: adjacent merges, hence >/<) + } else { + merged.0 = merged.0.min(s); + merged.1 = merged.1.max(e); + } + } + out.push(merged); + out.sort_unstable(); + self.intervals = out; + } + + /// True if `[0, n)` is covered by a single interval. + fn covers(&self, n: usize) -> bool { + matches!(self.intervals.first(), Some(&(0, e)) if e >= n) && self.intervals.len() == 1 + } +} + +/// A contiguous run of staged bytes ready to upload as one overlay chunk. +pub struct SealPiece { + /// Chunk index this run belongs to (for [`StreamingWriteBuffer::drop_chunk`]). + pub chunk_index: u64, + /// Absolute file offset of the run. + pub offset: u64, + /// The written bytes. + pub data: Vec, +} + +/// Per-file streaming write buffer. Stages dirty bytes by chunk index and tracks +/// the logical file size; the daemon seals chunks (uploads + [`drop_chunk`]s) to +/// keep resident memory bounded. +/// +/// [`drop_chunk`]: StreamingWriteBuffer::drop_chunk +pub struct StreamingWriteBuffer { + chunk_size: usize, + staged: BTreeMap, + size: u64, +} + +impl StreamingWriteBuffer { + pub fn new(chunk_size: usize) -> Self { + Self::with_size(chunk_size, 0) + } + + /// Construct with an initial logical size — the existing size of a file being + /// modified, so getattr/read report it before any write grows the buffer. + pub fn with_size(chunk_size: usize, size: u64) -> Self { + assert!(chunk_size > 0); + Self { + chunk_size, + staged: BTreeMap::new(), + size, + } + } + + /// Logical file size (max of all writes / truncates). + pub fn size(&self) -> u64 { + self.size + } + + /// Total bytes currently staged in memory (the resident-memory pressure). + pub fn staged_len(&self) -> usize { + self.staged.values().map(|c| c.data.len()).sum() + } + + /// No staged bytes remain (everything sealed/uploaded). + pub fn is_empty(&self) -> bool { + self.staged.is_empty() + } + + /// Stage `data` written at absolute `offset`, splitting across chunk + /// boundaries. Grows the logical size to cover the write. + pub fn write(&mut self, offset: u64, data: &[u8]) { + if data.is_empty() { + return; + } + let mut off = offset; + let mut rest = data; + let cs = self.chunk_size as u64; + while !rest.is_empty() { + let idx = off / cs; + let chunk_start = idx * cs; + let in_chunk = (off - chunk_start) as usize; + let take = rest.len().min(self.chunk_size - in_chunk); + let chunk = self.staged.entry(idx).or_insert_with(StagedChunk::new); + let end = in_chunk + take; + if chunk.data.len() < end { + chunk.data.resize(end, 0); + } + chunk.data[in_chunk..end].copy_from_slice(&rest[..take]); + chunk.mark(in_chunk, end); + off += take as u64; + rest = &rest[take..]; + } + self.size = self.size.max(offset.saturating_add(data.len() as u64)); + } + + /// Resize the logical file to `new_size`. Shrinking drops staged bytes beyond + /// the new end; growing leaves a hole (reads return zeros there). + pub fn truncate(&mut self, new_size: u64) { + if new_size < self.size { + let cs = self.chunk_size as u64; + self.staged.retain(|&idx, _| idx * cs < new_size); + if let Some((&idx, chunk)) = self.staged.range_mut(..).next_back() { + let chunk_start = idx * cs; + if chunk_start < new_size && new_size < chunk_start + cs { + let keep = (new_size - chunk_start) as usize; + if chunk.data.len() > keep { + chunk.data.truncate(keep); + } + chunk.intervals.retain_mut(|iv| { + iv.1 = iv.1.min(keep); + iv.0 < iv.1 + }); + } + } + } + self.size = new_size; + } + + /// Overlay still-staged bytes onto `out` (which the caller pre-filled with + /// committed data) for the read window starting at absolute `offset`. + pub fn overlay_read(&self, offset: u64, out: &mut [u8]) { + let want_start = offset; + let want_end = offset + out.len() as u64; + let cs = self.chunk_size as u64; + for (&idx, chunk) in &self.staged { + let chunk_start = idx * cs; + if chunk_start >= want_end || chunk_start + cs <= want_start { + continue; + } + for &(lo, hi) in &chunk.intervals { + let abs_lo = chunk_start + lo as u64; + let abs_hi = chunk_start + hi as u64; + let s = abs_lo.max(want_start); + let e = abs_hi.min(want_end); + if s < e { + let dst = (s - want_start) as usize..(e - want_start) as usize; + let src = (s - chunk_start) as usize..(e - chunk_start) as usize; + out[dst].copy_from_slice(&chunk.data[src]); + } + } + } + } + + /// Indices of chunks fully covering `[0, chunk_size)` — safe to seal eagerly + /// while writing so memory stays bounded. The final short chunk is excluded + /// (it only seals at flush via [`drain_all`]). + /// + /// [`drain_all`]: StreamingWriteBuffer::drain_all + pub fn full_chunk_indices(&self) -> Vec { + self.staged + .iter() + .filter(|(_, c)| c.covers(self.chunk_size)) + .map(|(&idx, _)| idx) + .collect() + } + + /// The lowest staged chunk index (for pressure-sealing a partial chunk). + pub fn lowest_index(&self) -> Option { + self.staged.keys().next().copied() + } + + /// Every staged chunk index (used on flush to seal all that remain). + pub fn staged_indices(&self) -> Vec { + self.staged.keys().copied().collect() + } + + /// Upload pieces for one chunk: one [`SealPiece`] per contiguous written + /// interval (so gaps are never zero-filled). Does NOT remove the chunk — + /// call [`drop_chunk`] once the uploads are confirmed, so a concurrent read + /// still sees the bytes while they are in flight. + /// + /// [`drop_chunk`]: StreamingWriteBuffer::drop_chunk + pub fn pieces_for(&self, idx: u64) -> Vec { + let cs = self.chunk_size as u64; + let chunk_start = idx * cs; + match self.staged.get(&idx) { + Some(chunk) => chunk + .intervals + .iter() + .map(|&(lo, hi)| SealPiece { + chunk_index: idx, + offset: chunk_start + lo as u64, + data: chunk.data[lo..hi].to_vec(), + }) + .collect(), + None => Vec::new(), + } + } + + /// Pieces for every staged chunk (used on flush to seal the partial tail). + pub fn drain_all(&self) -> Vec { + self.staged + .keys() + .flat_map(|&idx| self.pieces_for(idx)) + .collect() + } + + /// Forget a chunk's staged bytes once its pieces have been uploaded. + pub fn drop_chunk(&mut self, idx: u64) { + self.staged.remove(&idx); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const CS: usize = 4 * 1024 * 1024; + + #[test] + fn sequential_fills_seal_eagerly() { + let mut b = StreamingWriteBuffer::new(CS); + // Three full chunks written in 1 MiB pieces, then a short tail. + for i in 0..(3 * 4) { + b.write(i as u64 * (1 << 20), &vec![1u8; 1 << 20]); + } + b.write(3 * CS as u64, &vec![2u8; 1000]); + assert_eq!(b.size(), 3 * CS as u64 + 1000); + // Chunks 0,1,2 are full; the tail (chunk 3) is not. + assert_eq!(b.full_chunk_indices(), vec![0, 1, 2]); + } + + #[test] + fn full_chunk_pieces_are_single_run() { + let mut b = StreamingWriteBuffer::new(CS); + b.write(0, &vec![7u8; CS]); + let pieces = b.pieces_for(0); + assert_eq!(pieces.len(), 1); + assert_eq!(pieces[0].offset, 0); + assert_eq!(pieces[0].data.len(), CS); + } + + #[test] + fn sparse_chunk_uploads_only_written_intervals() { + let mut b = StreamingWriteBuffer::new(CS); + // Two disjoint runs in chunk 0, with a gap between them. + b.write(0, &[0xaa; 100]); + b.write(1_000_000, &[0xbb; 50]); + assert!(b.full_chunk_indices().is_empty()); // not fully covered + let pieces = b.pieces_for(0); + assert_eq!(pieces.len(), 2); + assert_eq!((pieces[0].offset, pieces[0].data.len()), (0, 100)); + assert_eq!((pieces[1].offset, pieces[1].data.len()), (1_000_000, 50)); + } + + #[test] + fn an_empty_write_past_the_end_keeps_the_size() { + let mut b = StreamingWriteBuffer::with_size(CS, 10); + b.write(1000, &[]); + assert_eq!(b.size(), 10); + assert!(b.is_empty()); + } + + #[test] + fn touching_writes_coalesce() { + let mut b = StreamingWriteBuffer::new(CS); + b.write(0, &[1u8; 100]); + b.write(100, &[1u8; 100]); // adjacent -> one interval + assert_eq!(b.pieces_for(0).len(), 1); + assert_eq!(b.pieces_for(0)[0].data.len(), 200); + } + + #[test] + fn overlay_read_overwrites_committed() { + let mut b = StreamingWriteBuffer::new(CS); + b.write(10, &[0xfe; 4]); + let mut out = vec![0u8; 20]; // pretend committed data is all zeros + b.overlay_read(0, &mut out); + let mut want = vec![0u8; 20]; + want[10..14].fill(0xfe); + assert_eq!(out, want); + } + + #[test] + fn overlay_read_partial_window() { + let mut b = StreamingWriteBuffer::new(CS); + b.write(CS as u64 - 2, &[9u8; 4]); // straddles chunk boundary + let mut out = vec![0u8; 4]; + b.overlay_read(CS as u64 - 2, &mut out); + assert_eq!(out, vec![9u8; 4]); + } + + #[test] + fn truncate_shrinks_and_drops() { + let mut b = StreamingWriteBuffer::new(CS); + b.write(0, &vec![1u8; CS + 100]); // chunk 0 full + 100 into chunk 1 + b.truncate(CS as u64 + 10); + assert_eq!(b.size(), CS as u64 + 10); + let p = b.pieces_for(1); + assert_eq!(p.len(), 1); + assert_eq!(p[0].data.len(), 10); + b.truncate(5); + assert!(b.pieces_for(1).is_empty()); + assert_eq!(b.pieces_for(0)[0].data.len(), 5); + } + + #[test] + fn drop_chunk_frees_memory() { + let mut b = StreamingWriteBuffer::new(CS); + b.write(0, &vec![1u8; CS]); + b.write(CS as u64, &vec![1u8; 1000]); + assert_eq!(b.staged_len(), CS + 1000); + b.drop_chunk(0); + assert_eq!(b.staged_len(), 1000); + assert_eq!(b.lowest_index(), Some(1)); + } +} diff --git a/seaweed-worker/crates/sw-client/src/write.rs b/seaweed-worker/crates/sw-client/src/write.rs new file mode 100644 index 0000000000..30dc7f868a --- /dev/null +++ b/seaweed-worker/crates/sw-client/src/write.rs @@ -0,0 +1,566 @@ +//! Data plane (write): upload file content as immutable chunks and persist the +//! chunk list on the filer. +//! +//! Phase B2 uses a simple whole-file model: the daemon buffers a file's content +//! and, on flush, re-uploads it as `chunk_size`-sized chunks and replaces the +//! entry's chunk list. Correct for sequential writes and full overwrites; an +//! incremental/append-merge pipeline is a later optimisation. + +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +use crate::filer::{AssignVolumeOptions, Assignment, FilerClient}; +use crate::filer_pb::{Entry, FileChunk}; +use crate::transport::{HttpTransport, VolumeTransport}; +use crate::wbuf::SealPiece; + +/// Current Unix time as (seconds, nanoseconds). +pub fn now_unix() -> (i64, i32) { + let d = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + (d.as_secs() as i64, d.subsec_nanos() as i32) +} + +/// Current Unix time in nanoseconds. +pub fn now_ns() -> i64 { + let d = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + d.as_nanos() as i64 +} + +/// Max chunk uploads in flight per flush (bounds load on the filer/volumes). +const MAX_PARALLEL_UPLOADS: usize = 8; + +/// Uploads chunk data to volume servers via a pluggable [`VolumeTransport`] +/// (HTTP by default; RDMA later). Volume assignment stays on the filer. +/// Cheap to clone (shares the transport) so a flush can upload chunks in parallel. +#[derive(Clone)] +pub struct Writer { + transport: Arc, +} + +impl Default for Writer { + fn default() -> Self { + Self::new() + } +} + +impl Writer { + pub fn new() -> Self { + Self::with_transport(Arc::new(HttpTransport::new())) + } + + /// Construct with a specific data-plane transport (e.g. an RDMA transport). + pub fn with_transport(transport: Arc) -> Self { + Self { transport } + } + + /// Assign a volume, upload `data` as one needle, and return the resulting + /// `FileChunk` positioned at `offset` in the file. + pub async fn upload_chunk( + &self, + client: &mut FilerClient, + file_path: &str, + filename: &str, + offset: i64, + data: &[u8], + ) -> Result { + self.upload_chunk_with_options( + client, + file_path, + filename, + offset, + data, + &AssignVolumeOptions::default(), + ) + .await + } + + pub async fn upload_chunk_with_options( + &self, + client: &mut FilerClient, + file_path: &str, + filename: &str, + offset: i64, + data: &[u8], + options: &AssignVolumeOptions, + ) -> Result { + self.upload_chunk_with_options_bytes( + client, + file_path, + filename, + offset, + Bytes::copy_from_slice(data), + options, + ) + .await + } + + pub async fn upload_chunk_with_options_bytes( + &self, + client: &mut FilerClient, + file_path: &str, + filename: &str, + offset: i64, + data: Bytes, + options: &AssignVolumeOptions, + ) -> Result { + let size = data.len() as u64; + let first = client + .assign_volume_with_options(file_path, size, options) + .await?; + upload_reassigning_once( + first, + data, + |assign, data| async move { + self.upload_assigned_chunk_bytes(&assign, filename, offset, data) + .await + }, + || client.assign_volume_with_options(file_path, size, options), + ) + .await + } + + /// Upload to a volume assignment that has already been resolved by the + /// filer. This is the data-plane seam: HTTP/RDMA details live only in the + /// transport implementation, while assignment and filer metadata stay above. + pub async fn upload_assigned_chunk( + &self, + assign: &Assignment, + filename: &str, + offset: i64, + data: &[u8], + ) -> Result { + self.upload_assigned_chunk_bytes(assign, filename, offset, Bytes::copy_from_slice(data)) + .await + } + + pub async fn upload_assigned_chunk_bytes( + &self, + assign: &Assignment, + filename: &str, + offset: i64, + data: Bytes, + ) -> Result { + self.upload_assigned_chunk_bytes_with_receipt(assign, filename, offset, data) + .await + .map(|(chunk, _)| chunk) + } + + pub async fn upload_assigned_chunk_bytes_with_receipt( + &self, + assign: &Assignment, + filename: &str, + offset: i64, + data: Bytes, + ) -> Result<(FileChunk, crate::VolumeWriteReceipt)> { + let size = data.len() as u64; + let receipt = self + .transport + .write_synced( + &assign.url, + &assign.file_id, + &assign.auth, + filename, + data, + assign.fsync, + ) + .await + .with_context(|| format!("upload {} to {}", assign.file_id, assign.url))?; + + Ok(( + FileChunk { + file_id: assign.file_id.clone(), + offset, + size, + modified_ts_ns: now_ns(), + ..Default::default() + }, + receipt, + )) + } +} + +/// Uploads under `first`; if the Volume refused before committing anything +/// (e.g. it was sealed for rotation after the assignment), asks `reassign` for +/// one fresh assignment and uploads once more. Any other result, a commit with +/// an unknown outcome included, returns as is, so nothing is written twice. +async fn upload_reassigning_once( + first: Assignment, + data: Bytes, + upload: U, + reassign: A, +) -> Result +where + U: Fn(Assignment, Bytes) -> UF, + UF: std::future::Future>, + A: FnOnce() -> AF, + AF: std::future::Future>, +{ + match upload(first, data.clone()).await { + Err(error) if crate::is_not_written(&error) => upload(reassign().await?, data).await, + other => other, + } +} + +/// Upload a batch of staged write pieces concurrently (bounded), each as one +/// overlay needle at its file offset. Returns the resulting `FileChunk`s sorted +/// by file offset. Used +/// by the streaming write path to seal chunks incrementally without holding the +/// whole file in memory. +pub async fn upload_pieces( + client: &FilerClient, + writer: &Writer, + file_path: &str, + filename: &str, + pieces: Vec, +) -> Result> { + upload_pieces_with_options( + client, + writer, + file_path, + filename, + pieces, + &AssignVolumeOptions::default(), + ) + .await +} + +pub async fn upload_pieces_with_options( + client: &FilerClient, + writer: &Writer, + file_path: &str, + filename: &str, + pieces: Vec, + options: &AssignVolumeOptions, +) -> Result> { + let sem = Arc::new(tokio::sync::Semaphore::new(MAX_PARALLEL_UPLOADS)); + let mut set = tokio::task::JoinSet::new(); + for p in pieces { + let permit = sem.clone().acquire_owned().await.expect("semaphore open"); + let mut c = client.clone(); + let w = writer.clone(); + let fp = file_path.to_string(); + let name = filename.to_string(); + let opts = options.clone(); + set.spawn(async move { + let _permit = permit; + w.upload_chunk_with_options_bytes( + &mut c, + &fp, + &name, + p.offset as i64, + Bytes::from(p.data), + &opts, + ) + .await + }); + } + let mut out = Vec::with_capacity(set.len()); + while let Some(joined) = set.join_next().await { + out.push(joined.context("piece upload task")??); + } + out.sort_by(|a, b| { + (a.offset, a.modified_ts_ns) + .cmp(&(b.offset, b.modified_ts_ns)) + .then_with(|| a.file_id.cmp(&b.file_id)) + }); + Ok(out) +} + +/// Flush a full-file buffer: upload it as `chunk_size` chunks and `UpdateEntry` +/// with the new chunk list and size/mtime. The entry is mutated in place. +/// +/// `data` is taken as a shared `Arc` (not a slice) so the caller's in-memory +/// write buffer is uploaded in place — no extra full-file copy. With a large +/// file this is the difference between ~1x and ~3x its size in resident memory. +pub async fn flush_file( + client: &mut FilerClient, + writer: &Writer, + directory: &str, + entry: &mut Entry, + data: Arc>, + chunk_size: usize, + signature: i32, +) -> Result<()> { + let file_path = if directory == "/" { + format!("/{}", entry.name) + } else { + format!("{directory}/{}", entry.name) + }; + + // Upload chunks concurrently (bounded), preserving order in the chunk list. + // FilerClient/Writer are cheap clones (h2 multiplexes), so parallel uploads + // overlap their AssignVolume + needle-POST round-trips. + anyhow::ensure!(chunk_size > 0, "flushing {file_path}: chunk size is zero"); + let n = data.len().div_ceil(chunk_size); + let shared = data; + let sem = Arc::new(tokio::sync::Semaphore::new(MAX_PARALLEL_UPLOADS)); + let mut set = tokio::task::JoinSet::new(); + for i in 0..n { + let start = i * chunk_size; + let end = (start + chunk_size).min(shared.len()); + let permit = sem.clone().acquire_owned().await.expect("semaphore open"); + let mut c = client.clone(); + let w = writer.clone(); + let d = shared.clone(); + let fp = file_path.clone(); + let name = entry.name.clone(); + set.spawn(async move { + let _permit = permit; + let r = w + .upload_chunk(&mut c, &fp, &name, start as i64, &d[start..end]) + .await; + (i, r) + }); + } + let mut slots: Vec> = (0..n).map(|_| None).collect(); + while let Some(joined) = set.join_next().await { + let (i, r) = joined.context("chunk upload task")?; + slots[i] = Some(r?); + } + + entry.chunks = slots + .into_iter() + .map(|c| c.expect("all chunks uploaded")) + .collect(); + // Small-file inline content from a previous version no longer applies. + entry.content = Vec::new(); + if let Some(a) = entry.attributes.as_mut() { + a.file_size = shared.len() as u64; + let (sec, nsec) = now_unix(); + a.mtime = sec; + a.mtime_ns = nsec; + } + + client + .update_entry(directory, entry.clone(), signature) + .await + .context("persisting chunk list")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + use std::sync::Mutex; + + fn assignment(fid: &str) -> Assignment { + Assignment { + file_id: fid.to_string(), + url: "10.0.2.3:19601".to_string(), + auth: String::new(), + fsync: false, + } + } + + fn chunk(fid: &str, data: &Bytes) -> FileChunk { + FileChunk { + file_id: fid.to_string(), + size: data.len() as u64, + ..Default::default() + } + } + + /// Runs the helper with an upload that answers from `answers` in order + /// and a reassign that returns `fresh`; reports the chunk and the calls. + async fn reassign_run( + answers: Vec<&'static str>, + fresh: Result, + ) -> (Result, Vec<(String, Vec)>, bool) { + let answers = Mutex::new(answers.into_iter()); + let uploads = Mutex::new(Vec::new()); + let reassigned = Mutex::new(false); + let result = upload_reassigning_once( + assignment("38,a1"), + Bytes::from_static(b"kv-bytes"), + |assign, data| { + uploads + .lock() + .unwrap() + .push((assign.file_id.clone(), data.to_vec())); + let answer = answers.lock().unwrap().next().unwrap(); + async move { + match answer { + "ok" => Ok(chunk(&assign.file_id, &data)), + "refused" => Err(anyhow::anyhow!("prepare rejected") + .context(crate::NotWritten) + .context("upload")), + _ => { + Err(anyhow::anyhow!("commit timed out").context(crate::CommitAmbiguous)) + } + } + } + }, + || { + *reassigned.lock().unwrap() = true; + async move { fresh } + }, + ) + .await; + let uploads = uploads.into_inner().unwrap(); + (result, uploads, reassigned.into_inner().unwrap()) + } + + #[tokio::test] + async fn a_refused_write_is_uploaded_once_more_under_a_fresh_assignment() { + let (result, uploads, reassigned) = + reassign_run(vec!["refused", "ok"], Ok(assignment("41,b2"))).await; + assert!(reassigned); + assert_eq!(result.unwrap().file_id, "41,b2"); + assert_eq!( + uploads, + vec![ + ("38,a1".to_string(), b"kv-bytes".to_vec()), + ("41,b2".to_string(), b"kv-bytes".to_vec()), + ] + ); + } + + #[tokio::test] + async fn a_zero_chunk_size_is_an_error() { + let mut client = FilerClient::connect_lazy("127.0.0.1:1").unwrap(); + let mut entry = Entry { + name: "f".into(), + ..Default::default() + }; + let err = flush_file( + &mut client, + &Writer::new(), + "/d", + &mut entry, + Arc::new(vec![1; 8]), + 0, + 0, + ) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("chunk size is zero"), "{err:#}"); + } + + #[tokio::test] + async fn the_retry_happens_at_most_once() { + let (result, uploads, _) = + reassign_run(vec!["refused", "refused"], Ok(assignment("41,b2"))).await; + assert!(crate::is_not_written(&result.unwrap_err())); + assert_eq!(uploads.len(), 2); + } + + #[tokio::test] + async fn an_ambiguous_commit_is_never_retried() { + let (result, uploads, reassigned) = + reassign_run(vec!["ambiguous"], Ok(assignment("41,b2"))).await; + assert!(result.is_err()); + assert!(!reassigned); + assert_eq!(uploads.len(), 1); + } + + #[tokio::test] + async fn a_failed_reassignment_returns_its_error_without_a_second_upload() { + let (result, uploads, reassigned) = + reassign_run(vec!["refused"], Err(anyhow::anyhow!("no writable volumes"))).await; + assert!(reassigned); + assert!( + result + .unwrap_err() + .to_string() + .contains("no writable volumes") + ); + assert_eq!(uploads.len(), 1); + } + + #[tokio::test] + async fn a_success_is_not_retried() { + let (result, uploads, reassigned) = reassign_run(vec!["ok"], Ok(assignment("41,b2"))).await; + assert_eq!(result.unwrap().file_id, "38,a1"); + assert!(!reassigned); + assert_eq!(uploads.len(), 1); + } + + #[derive(Debug, PartialEq, Eq)] + struct WriteCall { + location: String, + file_id: String, + auth: String, + filename: String, + data: Vec, + } + + #[derive(Default)] + struct MockTransport { + writes: Mutex>, + } + + #[async_trait] + impl VolumeTransport for MockTransport { + async fn read( + &self, + _locations: &[String], + _file_id: &str, + _range: Option<(u64, u64)>, + ) -> Result> { + anyhow::bail!("read is not used by write tests") + } + + async fn write( + &self, + location: &str, + file_id: &str, + auth: &str, + filename: &str, + data: Bytes, + ) -> Result<()> { + self.writes.lock().unwrap().push(WriteCall { + location: location.to_string(), + file_id: file_id.to_string(), + auth: auth.to_string(), + filename: filename.to_string(), + data: data.to_vec(), + }); + Ok(()) + } + + fn name(&self) -> &'static str { + "mock" + } + } + + #[tokio::test] + async fn assigned_chunk_upload_uses_volume_transport_only() { + let transport = Arc::new(MockTransport::default()); + let writer = Writer::with_transport(transport.clone()); + let assign = Assignment { + file_id: "7,abc".to_string(), + url: "10.0.0.2:8080".to_string(), + auth: "write-token".to_string(), + fsync: false, + }; + + let chunk = writer + .upload_assigned_chunk(&assign, "part-0", 4096, b"needle-bytes") + .await + .unwrap(); + + assert_eq!(chunk.file_id, "7,abc"); + assert_eq!(chunk.offset, 4096); + assert_eq!(chunk.size, 12); + + let writes = transport.writes.lock().unwrap(); + assert_eq!( + writes.as_slice(), + [WriteCall { + location: "10.0.0.2:8080".to_string(), + file_id: "7,abc".to_string(), + auth: "write-token".to_string(), + filename: "part-0".to_string(), + data: b"needle-bytes".to_vec(), + }] + ); + } +}