From af45ab8f2a26a00f16040b6b874124139a209e6b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 3 Nov 2026 04:53:57 -0800 Subject: [PATCH] s3api: known-answer fixture generator for SSE-S3 and filer-encrypted chunks (#12355) * s3api: a known-answer fixture generator for SSE-S3 and filer-encrypted chunks TestGenerateDirectReadFixture writes, from this package's own encryption, the pieces a reader outside the gateway must handle: SSE-S3 chunks with their per-chunk metadata, the object-level key, a single-stream ciphertext, the KEK derived from s3.sse.key and wrapped under a passphrase in both on-disk formats, and chunks encrypted by the filer, plain and gzipped. It is skipped unless DIRECT_READ_FIXTURE names the output file. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: the fixture generator restores the key manager and checks the write The test set the global key manager's KEK and left it; a later test in the same run then found a signing key where it expects none. The previous key is restored with t.Cleanup. The fixture file's close error is checked too, so a delayed write failure fails the test instead of leaving a short fixture. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: the fixture generator unwraps both KEK formats before writing The legacy v1 wrap is built by hand in the test; reading v1 and v2 back through unwrapKEK ties both to the format the key manager accepts. Co-Authored-By: Claude Opus 5.5 * s3api: pin a legacy v1 wrapped KEK made outside the package Generating the v1 sample through legacyKEKWrappingSalt let a change to the salt pass on both sides; a vector wrapped elsewhere under the historical salt must still unwrap. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 --- weed/s3api/direct_read_fixture_test.go | 159 ++++++++++++++++++++ weed/s3api/s3_sse_s3_kek_passphrase_test.go | 22 +++ 2 files changed, 181 insertions(+) create mode 100644 weed/s3api/direct_read_fixture_test.go diff --git a/weed/s3api/direct_read_fixture_test.go b/weed/s3api/direct_read_fixture_test.go new file mode 100644 index 0000000000..486432ca3b --- /dev/null +++ b/weed/s3api/direct_read_fixture_test.go @@ -0,0 +1,159 @@ +package s3api + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "os" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// Writes a known-answer fixture for readers that decode SSE-S3 objects and +// filer-encrypted chunks outside the gateway, from this package's own +// encryption: DIRECT_READ_FIXTURE=/path go test -run TestGenerateDirectReadFixture +func TestGenerateDirectReadFixture(t *testing.T) { + out := os.Getenv("DIRECT_READ_FIXTURE") + if out == "" { + t.Skip("set DIRECT_READ_FIXTURE to write the fixture") + } + kek, err := deriveKeyFromSecret("s3vectors-test") + if err != nil { + t.Fatal(err) + } + globalSSES3KeyManager.mu.Lock() + previous := globalSSES3KeyManager.superKey + globalSSES3KeyManager.superKey = kek + globalSSES3KeyManager.mu.Unlock() + t.Cleanup(func() { + globalSSES3KeyManager.mu.Lock() + globalSSES3KeyManager.superKey = previous + globalSSES3KeyManager.mu.Unlock() + }) + + plain := make([]byte, 5000) + for i := range plain { + plain[i] = byte(i*7 + i/13) + } + key, err := GenerateSSES3Key() + if err != nil { + t.Fatal(err) + } + baseIV := []byte("0123456789abcdef") + + type chunk struct { + Offset int64 `json:"offset"` + Ciphertext string `json:"ciphertext"` + SseMetadata string `json:"sse_metadata"` + } + var chunks []chunk + for _, part := range [][2]int64{{0, 2048}, {2048, 2952}} { + off, size := part[0], part[1] + enc, iv, err := CreateSSES3EncryptedReaderWithBaseIV(bytes.NewReader(plain[off:off+size]), key, baseIV, off) + if err != nil { + t.Fatal(err) + } + ct, err := io.ReadAll(enc) + if err != nil { + t.Fatal(err) + } + meta, err := SerializeSSES3Metadata(&SSES3Key{Key: key.Key, KeyID: key.KeyID, Algorithm: key.Algorithm, IV: iv}) + if err != nil { + t.Fatal(err) + } + chunks = append(chunks, chunk{off, hex.EncodeToString(ct), string(meta)}) + } + objectMeta, err := SerializeSSES3Metadata(&SSES3Key{Key: key.Key, KeyID: key.KeyID, Algorithm: key.Algorithm, IV: baseIV}) + if err != nil { + t.Fatal(err) + } + encWhole, _, err := CreateSSES3EncryptedReaderWithBaseIV(bytes.NewReader(plain), key, baseIV, 0) + if err != nil { + t.Fatal(err) + } + whole, err := io.ReadAll(encWhole) + if err != nil { + t.Fatal(err) + } + + ck := util.GenCipherKey() + cipherPlain, err := util.Encrypt(plain, ck) + if err != nil { + t.Fatal(err) + } + gz, err := util.GzipData(plain) + if err != nil { + t.Fatal(err) + } + cipherGzip, err := util.Encrypt(gz, ck) + if err != nil { + t.Fatal(err) + } + + km := NewSSES3KeyManager("pass-phrase") + v2, err := km.wrapKEK(kek) + if err != nil { + t.Fatal(err) + } + wrappingKey, err := km.deriveWrappingKey(legacyKEKWrappingSalt) + if err != nil { + t.Fatal(err) + } + block, err := aes.NewCipher(wrappingKey) + if err != nil { + t.Fatal(err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + t.Fatal(err) + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + t.Fatal(err) + } + v1 := base64.StdEncoding.EncodeToString(gcm.Seal(nonce, nonce, kek, nil)) + for _, wrapped := range []struct { + data []byte + isV2 bool + }{{[]byte(v1), false}, {v2, true}} { + unwrapped, isV2, err := km.unwrapKEK(wrapped.data) + if err != nil || isV2 != wrapped.isV2 || !bytes.Equal(unwrapped, kek) { + t.Fatalf("unwrapKEK(v2=%v) = v2=%v, err=%v; want the KEK back", wrapped.isV2, isV2, err) + } + } + + fixture := map[string]any{ + "kek_secret": "s3vectors-test", + "kek_hex": hex.EncodeToString(kek), + "kek_passphrase": "pass-phrase", + "kek_wrapped_v1": v1, + "kek_wrapped_v2": string(v2), + "plaintext": hex.EncodeToString(plain), + "object_metadata": string(objectMeta), + "chunks": chunks, + "whole_ciphertext": hex.EncodeToString(whole), + "cipher_key": hex.EncodeToString(ck), + "cipher_plain": hex.EncodeToString(cipherPlain), + "cipher_gzip": hex.EncodeToString(cipherGzip), + "commitment_sample": hex.EncodeToString(ComputeKeyCommitment(key.Key, baseIV, key.Algorithm)), + } + f, err := os.Create(out) + if err != nil { + t.Fatal(err) + } + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(fixture); err != nil { + f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/weed/s3api/s3_sse_s3_kek_passphrase_test.go b/weed/s3api/s3_sse_s3_kek_passphrase_test.go index ea9f25c0fa..9d930be951 100644 --- a/weed/s3api/s3_sse_s3_kek_passphrase_test.go +++ b/weed/s3api/s3_sse_s3_kek_passphrase_test.go @@ -87,3 +87,25 @@ func TestNoPassphraseKeepsLegacyHexDecodePath(t *testing.T) { t.Fatal("wrapKEK without passphrase should fail; got nil error") } } + +// A KEK wrapped in the legacy v1 layout (nonce || ciphertext+tag) under the +// fixed salt "seaweedfs-sse-s3-kek-wrapping-v1", produced outside this +// package, so a change to legacyKEKWrappingSalt cannot pass by changing +// both sides. +func TestUnwrapKEKReadsPinnedLegacyV1(t *testing.T) { + km := NewSSES3KeyManager("legacy-passphrase") + got, isV2, err := km.unwrapKEK([]byte("oKGio6SlpqeoqaqrixOn/dNaewpX3OoMjLT3VUFh+zg+Hd1fqbiWa6Xhc3SAg6hZdXb+Pwo520Fc1dfj")) + if err != nil { + t.Fatalf("unwrapKEK: %v", err) + } + if isV2 { + t.Fatal("a legacy payload was reported as v2") + } + want := make([]byte, SSES3KeySize) + for i := range want { + want[i] = byte(i) + } + if string(got) != string(want) { + t.Fatalf("unwrapped %x, want %x", got, want) + } +}