mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-03 02:32:35 +01:00
s3vectors: encrypt the objects of a vector bucket declared AES256 (#12324)
* s3api: encrypt the objects of a vector bucket declared AES256 CreateVectorBucket recorded encryptionConfiguration sseType AES256 and returned it, but nothing on the write path read it: the index's data files, manifests and transactions were stored without SSE. A LANCE table bucket's record now supplies the bucket's default encryption whenever the bucket has none of its own, so PutObject, CreateMultipartUpload and CopyObject apply SSE-S3 to every object the engine writes. The S3 bucket metadata itself is unchanged. ICEBERG table buckets keep recording the declaration only: their catalog reads the manifests clients upload straight from the filer, which would see ciphertext. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3tables: refuse an AES256 vector bucket on a gateway without an SSE-S3 key A vector bucket's AES256 is now applied to every index write, and SSE-S3 fails each write when the gateway has no KEK. Creating the bucket there succeeded and the first CreateIndex came back as InternalFailure. CreateTableBucket now refuses a LANCE bucket declaring AES256 when the gateway reports no SSE-S3 key, naming s3.sse.kek and s3.sse.key; CreateVectorBucket returns it as ValidationException. ICEBERG buckets, whose declaration is only recorded, and tooling that sets no checker are unaffected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: refuse DeleteBucketEncryption on a bucket's declared encryption GetBucketEncryption reports a vector bucket's declared AES256, but DeleteBucketEncryption answered NoSuchBucketEncryptionConfiguration and the declaration kept applying. The declaration is fixed when the bucket is created, so the delete now returns MethodNotAllowed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): keep an inherited WEED_S3_SSE_KEK out of the test cluster The gateway refuses to start SSE-S3 with both s3.sse.kek and s3.sse.key set, so a developer shell exporting WEED_S3_SSE_KEK made the AES256 bucket test fail. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d72f761d83
commit
2eb219df61
15 files changed
+249
-3
No files matched your search
@@ -2,6 +2,7 @@ package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// Issue 11647: Bucket default encryption (SSE-S3) must be applied when
|
||||
@@ -218,3 +220,72 @@ func TestPutObjectUnderUploadsFolderKeepsBucketDefaultEncryption(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A vector bucket is a LANCE table bucket; the AES256 its record declares is
|
||||
// the bucket's default encryption. An ICEBERG bucket's declaration is not
|
||||
// applied, because its catalog reads client manifests straight from the filer.
|
||||
func TestPutObjectAppliesTableBucketDeclaredEncryption(t *testing.T) {
|
||||
km := GetSSES3KeyManager()
|
||||
oldSuperKey := km.superKey
|
||||
km.superKey = make([]byte, 32)
|
||||
for i := range km.superKey {
|
||||
km.superKey[i] = byte(i + 1)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
km.superKey = oldSuperKey
|
||||
})
|
||||
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{
|
||||
volume: volume,
|
||||
entries: map[string]*filer_pb.Entry{},
|
||||
apply: true,
|
||||
}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
for bucket, format := range map[string]string{"lance": s3tables.FormatLance, "ice": s3tables.FormatIceberg} {
|
||||
s3a.bucketConfigCache.Set(bucket, s3a.newBucketConfigFromEntry(bucket, &filer_pb.Entry{
|
||||
Name: bucket,
|
||||
Extended: map[string][]byte{
|
||||
s3tables.ExtendedKeyTableBucket: []byte("true"),
|
||||
s3tables.ExtendedKeyMetadata: []byte(`{"name":"` + bucket + `","format":"` + format +
|
||||
`","encryptionConfiguration":{"sseAlgorithm":"AES256"}}`),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
cfg, err := s3a.GetBucketEncryptionConfig("lance")
|
||||
if err != nil || cfg.SseAlgorithm != EncryptionTypeAES256 {
|
||||
t.Fatalf("GetBucketEncryptionConfig(lance) = %v, %v; want AES256", cfg, err)
|
||||
}
|
||||
if metadata, err := s3a.GetBucketMetadata("lance"); err != nil || metadata.Encryption != nil {
|
||||
t.Fatalf("declared encryption leaked into the stored bucket metadata: %v, %v", metadata, err)
|
||||
}
|
||||
if _, err := s3a.GetBucketEncryptionConfig("ice"); !errors.Is(err, ErrNoEncryptionConfig) {
|
||||
t.Fatalf("GetBucketEncryptionConfig(ice) error = %v, want ErrNoEncryptionConfig", err)
|
||||
}
|
||||
if code := s3a.removeEncryptionConfiguration("lance"); code != s3err.ErrMethodNotAllowed {
|
||||
t.Fatalf("DeleteBucketEncryption(lance) = %v, want MethodNotAllowed for a declared encryption", code)
|
||||
}
|
||||
if code := s3a.removeEncryptionConfiguration("ice"); code != s3err.ErrNoSuchBucketEncryptionConfiguration {
|
||||
t.Fatalf("DeleteBucketEncryption(ice) = %v, want NoSuchBucketEncryptionConfiguration", code)
|
||||
}
|
||||
|
||||
for bucket, want := range map[string]filer_pb.SSEType{"lance": filer_pb.SSEType_SSE_S3, "ice": filer_pb.SSEType_NONE} {
|
||||
filePath := "/buckets/" + bucket + "/vector/idx/data/0.lance"
|
||||
r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/vector/idx/data/0.lance", nil)
|
||||
_, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("lance data file"), bucket, "vector/idx/data/0.lance", 1, 0, nil, false, "")
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("putToFiler(%s) = %v, want %v", bucket, code, s3err.ErrNone)
|
||||
}
|
||||
entry := filerImpl.entries[filePath]
|
||||
if entry == nil || len(entry.Chunks) == 0 {
|
||||
t.Fatalf("no entry with chunks at %s", filePath)
|
||||
}
|
||||
for i, chunk := range entry.Chunks {
|
||||
if chunk.SseType != want {
|
||||
t.Errorf("%s chunk %d: SseType %v, want %v", bucket, i, chunk.SseType, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user