mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-02 18:22:40 +01:00
s3vectors: encrypt the objects of a vector bucket declared AES256 (#12324)
* s3api: encrypt the objects of a vector bucket declared AES256 CreateVectorBucket recorded encryptionConfiguration sseType AES256 and returned it, but nothing on the write path read it: the index's data files, manifests and transactions were stored without SSE. A LANCE table bucket's record now supplies the bucket's default encryption whenever the bucket has none of its own, so PutObject, CreateMultipartUpload and CopyObject apply SSE-S3 to every object the engine writes. The S3 bucket metadata itself is unchanged. ICEBERG table buckets keep recording the declaration only: their catalog reads the manifests clients upload straight from the filer, which would see ciphertext. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3tables: refuse an AES256 vector bucket on a gateway without an SSE-S3 key A vector bucket's AES256 is now applied to every index write, and SSE-S3 fails each write when the gateway has no KEK. Creating the bucket there succeeded and the first CreateIndex came back as InternalFailure. CreateTableBucket now refuses a LANCE bucket declaring AES256 when the gateway reports no SSE-S3 key, naming s3.sse.kek and s3.sse.key; CreateVectorBucket returns it as ValidationException. ICEBERG buckets, whose declaration is only recorded, and tooling that sets no checker are unaffected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: refuse DeleteBucketEncryption on a bucket's declared encryption GetBucketEncryption reports a vector bucket's declared AES256, but DeleteBucketEncryption answered NoSuchBucketEncryptionConfiguration and the declaration kept applying. The declaration is fixed when the bucket is created, so the delete now returns MethodNotAllowed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): keep an inherited WEED_S3_SSE_KEK out of the test cluster The gateway refuses to start SSE-S3 with both s3.sse.kek and s3.sse.key set, so a developer shell exporting WEED_S3_SSE_KEK made the AES256 bucket test fail. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d72f761d83
commit
2eb219df61
15 files changed
+249
-3
No files matched your search
@@ -102,7 +102,9 @@ def run_cluster(param, tmp_path_factory, identities):
|
||||
def start(name, args, unset=()):
|
||||
log = stack.enter_context((directory / f"{name}.log").open("w"))
|
||||
env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test",
|
||||
AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true")
|
||||
AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true",
|
||||
WEED_S3_SSE_KEY="s3vectors-test")
|
||||
env.pop("WEED_S3_SSE_KEK", None)
|
||||
for key in unset:
|
||||
env.pop(key)
|
||||
process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Run the S3 Vectors API against Go/Rust volumes and single/distributed queries."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
@@ -297,17 +298,44 @@ def test_vector_policy_permissions(vectors, vector_clients, index):
|
||||
vectors.delete_index(**other)
|
||||
|
||||
|
||||
def index_object_encryption(vectors, bucket):
|
||||
s3 = boto3.client("s3", endpoint_url=vectors.meta.endpoint_url, aws_access_key_id="test",
|
||||
aws_secret_access_key="test", region_name="us-east-1")
|
||||
keys = [o["Key"] for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=f"vector/{INDEX}/")
|
||||
for o in page.get("Contents", []) if o["Size"] > 0]
|
||||
assert any("/data/" in key for key in keys) and any("/_versions/" in key for key in keys)
|
||||
return {key: s3.head_object(Bucket=bucket, Key=key).get("ServerSideEncryption") for key in keys}
|
||||
|
||||
|
||||
def test_vector_bucket_encryption(vectors):
|
||||
bucket = "encryption-" + uuid.uuid4().hex[:16]
|
||||
plain = bucket + "-plain"
|
||||
config = {"sseType": "AES256"}
|
||||
vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config)
|
||||
vectors.create_vector_bucket(vectorBucketName=plain)
|
||||
try:
|
||||
assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config
|
||||
expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms",
|
||||
encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"})
|
||||
assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == []
|
||||
|
||||
for name in (bucket, plain):
|
||||
vectors.create_index(vectorBucketName=name, indexName=INDEX, dataType="float32", dimension=4,
|
||||
distanceMetric="euclidean")
|
||||
vectors.put_vectors(vectorBucketName=name, indexName=INDEX, vectors=[
|
||||
{"key": "a", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"color": "red"}},
|
||||
{"key": "b", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"color": "blue"}},
|
||||
])
|
||||
hits = vectors.query_vectors(vectorBucketName=name, indexName=INDEX, queryVector={"float32": [1, 0, 0, 0]},
|
||||
topK=1, returnMetadata=True)["vectors"]
|
||||
assert [(h["key"], h["metadata"]) for h in hits] == [("a", {"color": "red"})]
|
||||
assert set(index_object_encryption(vectors, bucket).values()) == {"AES256"}
|
||||
assert set(index_object_encryption(vectors, plain).values()) == {None}
|
||||
finally:
|
||||
vectors.delete_vector_bucket(vectorBucketName=bucket)
|
||||
for name in (bucket, plain):
|
||||
with contextlib.suppress(ClientError):
|
||||
vectors.delete_index(vectorBucketName=name, indexName=INDEX)
|
||||
vectors.delete_vector_bucket(vectorBucketName=name)
|
||||
|
||||
|
||||
def test_catalog_pagination(vectors, index):
|
||||
|
||||
@@ -724,6 +724,7 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp
|
||||
vectorsServer := s3vectors.NewServer(s3ApiServer, s3ApiServer, backend)
|
||||
vectorsServer.SetRegion(s3tables.DefaultRegion)
|
||||
vectorsServer.SetAccountID(s3tables.DefaultAccountID)
|
||||
vectorsServer.SetEncryptionChecker(s3api.GetSSES3KeyManager().IsConfigured)
|
||||
vectorsServer.RegisterRoutes(vectorsRouter)
|
||||
return vectorsRouter
|
||||
}
|
||||
|
||||
@@ -213,12 +213,28 @@ func (s3a *S3ApiServer) getEncryptionConfiguration(bucket string) (*s3_pb.Encryp
|
||||
}
|
||||
|
||||
if metadata.Encryption == nil {
|
||||
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
return s3a.declaredEncryption(bucket)
|
||||
}
|
||||
|
||||
return metadata.Encryption, s3err.ErrNone
|
||||
}
|
||||
|
||||
// declaredEncryption is the encryption a table bucket's record declares. It
|
||||
// applies while the bucket has no S3 default encryption of its own.
|
||||
func (s3a *S3ApiServer) declaredEncryption(bucket string) (*s3_pb.EncryptionConfiguration, s3err.ErrorCode) {
|
||||
if s3a.bucketConfigCache == nil {
|
||||
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
}
|
||||
config, errCode := s3a.getBucketConfig(bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
return nil, errCode
|
||||
}
|
||||
if config.DeclaredEncryption == nil {
|
||||
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
}
|
||||
return config.DeclaredEncryption, s3err.ErrNone
|
||||
}
|
||||
|
||||
// updateEncryptionConfiguration updates the encryption configuration for a bucket
|
||||
func (s3a *S3ApiServer) updateEncryptionConfiguration(bucket string, encryptionConfig *s3_pb.EncryptionConfiguration) s3err.ErrorCode {
|
||||
// Update using structured API
|
||||
@@ -249,6 +265,10 @@ func (s3a *S3ApiServer) removeEncryptionConfiguration(bucket string) s3err.Error
|
||||
}
|
||||
|
||||
if metadata.Encryption == nil {
|
||||
// A bucket's declared encryption is fixed when the bucket is created.
|
||||
if declared, _ := s3a.declaredEncryption(bucket); declared != nil {
|
||||
return s3err.ErrMethodNotAllowed
|
||||
}
|
||||
return s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
}
|
||||
|
||||
|
||||
@@ -778,6 +778,13 @@ func (km *SSES3KeyManager) StoreKey(key *SSES3Key) {
|
||||
// The DEK is encrypted with the super key and stored in object metadata
|
||||
}
|
||||
|
||||
// IsConfigured reports whether a KEK is loaded, i.e. SSE-S3 can encrypt.
|
||||
func (km *SSES3KeyManager) IsConfigured() bool {
|
||||
km.mu.RLock()
|
||||
defer km.mu.RUnlock()
|
||||
return len(km.superKey) > 0
|
||||
}
|
||||
|
||||
// GetMasterKey returns a derived key from the master KEK for STS signing
|
||||
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
|
||||
func (km *SSES3KeyManager) GetMasterKey() []byte {
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// BucketConfig represents cached bucket configuration. Only fields parsed
|
||||
@@ -56,6 +57,7 @@ type BucketConfig struct {
|
||||
LifecycleTransitionMinSize string
|
||||
Tags map[string]string // parsed from entry content
|
||||
Encryption *s3_pb.EncryptionConfiguration // parsed from entry content
|
||||
DeclaredEncryption *s3_pb.EncryptionConfiguration // declared by a table bucket's record
|
||||
KMSKeyCache *BucketKMSCache // Per-bucket KMS key cache for SSE-KMS operations
|
||||
LastModified time.Time
|
||||
}
|
||||
@@ -441,6 +443,9 @@ func (s3a *S3ApiServer) newBucketConfigFromEntry(bucket string, entry *filer_pb.
|
||||
config.ObjectLockConfig = objectLockConfig
|
||||
}
|
||||
config.BucketPolicy = loadBucketPolicyFromExtended(entry, bucket)
|
||||
if algorithm := s3tables.DeclaredBucketEncryption(entry); algorithm != "" {
|
||||
config.DeclaredEncryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: algorithm}
|
||||
}
|
||||
|
||||
if lifecycleXML, exists := entry.Extended[bucketLifecycleConfigurationXMLKey]; exists && len(lifecycleXML) > 0 {
|
||||
config.LifecycleXML = lifecycleXML
|
||||
|
||||
@@ -2,6 +2,7 @@ package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// Issue 11647: Bucket default encryption (SSE-S3) must be applied when
|
||||
@@ -218,3 +220,72 @@ func TestPutObjectUnderUploadsFolderKeepsBucketDefaultEncryption(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A vector bucket is a LANCE table bucket; the AES256 its record declares is
|
||||
// the bucket's default encryption. An ICEBERG bucket's declaration is not
|
||||
// applied, because its catalog reads client manifests straight from the filer.
|
||||
func TestPutObjectAppliesTableBucketDeclaredEncryption(t *testing.T) {
|
||||
km := GetSSES3KeyManager()
|
||||
oldSuperKey := km.superKey
|
||||
km.superKey = make([]byte, 32)
|
||||
for i := range km.superKey {
|
||||
km.superKey[i] = byte(i + 1)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
km.superKey = oldSuperKey
|
||||
})
|
||||
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{
|
||||
volume: volume,
|
||||
entries: map[string]*filer_pb.Entry{},
|
||||
apply: true,
|
||||
}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
for bucket, format := range map[string]string{"lance": s3tables.FormatLance, "ice": s3tables.FormatIceberg} {
|
||||
s3a.bucketConfigCache.Set(bucket, s3a.newBucketConfigFromEntry(bucket, &filer_pb.Entry{
|
||||
Name: bucket,
|
||||
Extended: map[string][]byte{
|
||||
s3tables.ExtendedKeyTableBucket: []byte("true"),
|
||||
s3tables.ExtendedKeyMetadata: []byte(`{"name":"` + bucket + `","format":"` + format +
|
||||
`","encryptionConfiguration":{"sseAlgorithm":"AES256"}}`),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
cfg, err := s3a.GetBucketEncryptionConfig("lance")
|
||||
if err != nil || cfg.SseAlgorithm != EncryptionTypeAES256 {
|
||||
t.Fatalf("GetBucketEncryptionConfig(lance) = %v, %v; want AES256", cfg, err)
|
||||
}
|
||||
if metadata, err := s3a.GetBucketMetadata("lance"); err != nil || metadata.Encryption != nil {
|
||||
t.Fatalf("declared encryption leaked into the stored bucket metadata: %v, %v", metadata, err)
|
||||
}
|
||||
if _, err := s3a.GetBucketEncryptionConfig("ice"); !errors.Is(err, ErrNoEncryptionConfig) {
|
||||
t.Fatalf("GetBucketEncryptionConfig(ice) error = %v, want ErrNoEncryptionConfig", err)
|
||||
}
|
||||
if code := s3a.removeEncryptionConfiguration("lance"); code != s3err.ErrMethodNotAllowed {
|
||||
t.Fatalf("DeleteBucketEncryption(lance) = %v, want MethodNotAllowed for a declared encryption", code)
|
||||
}
|
||||
if code := s3a.removeEncryptionConfiguration("ice"); code != s3err.ErrNoSuchBucketEncryptionConfiguration {
|
||||
t.Fatalf("DeleteBucketEncryption(ice) = %v, want NoSuchBucketEncryptionConfiguration", code)
|
||||
}
|
||||
|
||||
for bucket, want := range map[string]filer_pb.SSEType{"lance": filer_pb.SSEType_SSE_S3, "ice": filer_pb.SSEType_NONE} {
|
||||
filePath := "/buckets/" + bucket + "/vector/idx/data/0.lance"
|
||||
r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/vector/idx/data/0.lance", nil)
|
||||
_, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("lance data file"), bucket, "vector/idx/data/0.lance", 1, 0, nil, false, "")
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("putToFiler(%s) = %v, want %v", bucket, code, s3err.ErrNone)
|
||||
}
|
||||
entry := filerImpl.entries[filePath]
|
||||
if entry == nil || len(entry.Chunks) == 0 {
|
||||
t.Fatalf("no entry with chunks at %s", filePath)
|
||||
}
|
||||
for i, chunk := range entry.Chunks {
|
||||
if chunk.SseType != want {
|
||||
t.Errorf("%s chunk %d: SseType %v, want %v", bucket, i, chunk.SseType, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,11 @@ func (st *S3TablesApiServer) SetLocalAccountChecker(fn func(string) bool) {
|
||||
st.handler.SetLocalAccountChecker(fn)
|
||||
}
|
||||
|
||||
// SetEncryptionChecker sets the handler's SSE-S3 availability predicate.
|
||||
func (st *S3TablesApiServer) SetEncryptionChecker(fn func() bool) {
|
||||
st.handler.SetEncryptionChecker(fn)
|
||||
}
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
|
||||
st.handler.HandleRequest(w, r, st)
|
||||
@@ -72,6 +77,7 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p
|
||||
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
|
||||
// Create S3 Tables handler
|
||||
s3TablesApi := NewS3TablesApiServer(s3a)
|
||||
s3TablesApi.SetEncryptionChecker(GetSSES3KeyManager().IsConfigured)
|
||||
if s3a.iam != nil {
|
||||
s3TablesApi.SetLocalAccountChecker(s3a.iam.isConfiguredAccountID)
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ type S3TablesHandler struct {
|
||||
trusted bool // Trusted local tooling (shell/admin) bypasses authorization
|
||||
iamAuthorizer IAMAuthorizer
|
||||
localAccountIDs func(string) bool
|
||||
canEncrypt func() bool
|
||||
}
|
||||
|
||||
// NewS3TablesHandler creates a new S3 Tables handler
|
||||
@@ -119,6 +120,13 @@ func (h *S3TablesHandler) SetLocalAccountChecker(fn func(string) bool) {
|
||||
h.localAccountIDs = fn
|
||||
}
|
||||
|
||||
// SetEncryptionChecker injects the predicate that tells whether the gateway
|
||||
// holds an SSE-S3 key, so a bucket whose declared AES256 the write path
|
||||
// applies is not created where every write would then fail.
|
||||
func (h *S3TablesHandler) SetEncryptionChecker(fn func() bool) {
|
||||
h.canEncrypt = fn
|
||||
}
|
||||
|
||||
// FilerClient interface for filer operations
|
||||
type FilerClient interface {
|
||||
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
|
||||
|
||||
@@ -80,6 +80,12 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http
|
||||
}
|
||||
}
|
||||
|
||||
if len(req.EncryptionConfiguration) > 0 && IsCatalogOnlyFormat(bucketFormat) && h.canEncrypt != nil && !h.canEncrypt() {
|
||||
msg := "AES256 encryption needs an SSE-S3 key on the S3 gateway: set s3.sse.kek or s3.sse.key in security.toml"
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, msg)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(req.Name)
|
||||
|
||||
// Check if bucket already exists and ensure no conflict with object store buckets
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -107,3 +109,39 @@ func TestNormalizeFormat(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeclaredBucketEncryption(t *testing.T) {
|
||||
entry := func(format string, cfg map[string]interface{}) *filer_pb.Entry {
|
||||
meta, _ := json.Marshal(tableBucketMetadata{Name: "b", Format: format, EncryptionConfiguration: cfg})
|
||||
return &filer_pb.Entry{Extended: map[string][]byte{ExtendedKeyTableBucket: []byte("true"), ExtendedKeyMetadata: meta}}
|
||||
}
|
||||
aes := map[string]interface{}{"sseAlgorithm": "AES256"}
|
||||
|
||||
require.Equal(t, "AES256", DeclaredBucketEncryption(entry(FormatLance, aes)))
|
||||
require.Empty(t, DeclaredBucketEncryption(entry(FormatLance, nil)))
|
||||
require.Empty(t, DeclaredBucketEncryption(entry(FormatIceberg, aes)), "the Iceberg catalog reads client manifests raw")
|
||||
require.Empty(t, DeclaredBucketEncryption(entry("", aes)), "a bucket without a format is Iceberg")
|
||||
require.Empty(t, DeclaredBucketEncryption(&filer_pb.Entry{Extended: map[string][]byte{ExtendedKeyMetadata: entry(FormatLance, aes).Extended[ExtendedKeyMetadata]}}), "not a table bucket")
|
||||
require.Empty(t, DeclaredBucketEncryption(nil))
|
||||
}
|
||||
|
||||
// Only a format whose files the catalog leaves to clients gets its declared
|
||||
// AES256 applied, so only that format needs the gateway's SSE-S3 key.
|
||||
func TestCreateTableBucketNeedsSSES3ForADeclaredLanceBucket(t *testing.T) {
|
||||
fs := s3tablestest.Start(t)
|
||||
m := NewManager()
|
||||
m.SetEncryptionChecker(func() bool { return false })
|
||||
create := func(name, format string, cfg map[string]interface{}) error {
|
||||
return m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateTableBucket",
|
||||
&CreateTableBucketRequest{Name: name, Format: format, EncryptionConfiguration: cfg}, nil, s3_constants.AccountAdminId)
|
||||
}
|
||||
aes := map[string]interface{}{"sseAlgorithm": "AES256"}
|
||||
|
||||
err := create("lance-sealed", FormatLance, aes)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
require.Equal(t, ErrCodeInvalidRequest, s3Err.Type)
|
||||
require.Contains(t, s3Err.Message, "s3.sse.kek")
|
||||
require.NoError(t, create("lance-plain", FormatLance, nil))
|
||||
require.NoError(t, create("iceberg-sealed", FormatIceberg, aes), "an Iceberg bucket only records the declaration")
|
||||
}
|
||||
@@ -46,6 +46,11 @@ func (m *Manager) SetTrusted(trusted bool) {
|
||||
m.handler.SetTrusted(trusted)
|
||||
}
|
||||
|
||||
// SetEncryptionChecker sets the handler's SSE-S3 availability predicate.
|
||||
func (m *Manager) SetEncryptionChecker(fn func() bool) {
|
||||
m.handler.SetEncryptionChecker(fn)
|
||||
}
|
||||
|
||||
// PrincipalFor resolves the caller of r the way the catalog's handlers do,
|
||||
// so it matches the owner CreateTableBucket records. A nil request is the
|
||||
// handler's account ID.
|
||||
|
||||
@@ -3,6 +3,7 @@ package s3tables
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -192,6 +193,24 @@ func ValidateEncryptionConfiguration(cfg map[string]interface{}, sseField string
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeclaredBucketEncryption returns the sseAlgorithm a table bucket's record
|
||||
// declares, for the S3 write path to apply, when the bucket's format leaves
|
||||
// every file to its clients. An ICEBERG bucket gets "": its catalog reads the
|
||||
// manifests clients upload straight from the filer, past S3 decryption.
|
||||
func DeclaredBucketEncryption(entry *filer_pb.Entry) string {
|
||||
if !IsTableBucketEntry(entry) {
|
||||
return ""
|
||||
}
|
||||
var metadata tableBucketMetadata
|
||||
if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil || !IsCatalogOnlyFormat(metadata.Format) {
|
||||
return ""
|
||||
}
|
||||
if algorithm, _ := metadata.EncryptionConfiguration["sseAlgorithm"].(string); algorithm == "AES256" {
|
||||
return algorithm
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseS3Location(location string) (bucket, tablePath string, err error) {
|
||||
if !strings.HasPrefix(location, "s3://") {
|
||||
return "", "", fmt.Errorf("unsupported location: %s", location)
|
||||
|
||||
@@ -704,6 +704,32 @@ func TestValidateEncryptionConfigurationRejectsCustomerKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The gateway applies a vector bucket's AES256 to every index write, so a
|
||||
// gateway without an SSE-S3 key refuses the bucket instead of failing each
|
||||
// later write.
|
||||
func TestCreateVectorBucketNeedsSSES3ForAES256(t *testing.T) {
|
||||
fs := s3tablestest.Start(t)
|
||||
s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0))
|
||||
keyLoaded := false
|
||||
s.SetEncryptionChecker(func() bool { return keyLoaded })
|
||||
aes := map[string]interface{}{"sseType": "AES256"}
|
||||
|
||||
w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "sealed", EncryptionConfiguration: aes})
|
||||
if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") || !strings.Contains(w.Body.String(), "s3.sse.key") {
|
||||
t.Fatalf("AES256 without a key: %d %s, want 400 ValidationException naming s3.sse.key", w.Code, w.Body.String())
|
||||
}
|
||||
if w := postJSON(t, s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "sealed"}); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("refused bucket exists: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "plain"}); w.Code != http.StatusOK {
|
||||
t.Fatalf("unencrypted bucket without a key: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
keyLoaded = true
|
||||
if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "sealed", EncryptionConfiguration: aes}); w.Code != http.StatusOK {
|
||||
t.Fatalf("AES256 with a key: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A vector index is colocated: its catalog entry sits at the dataset root, so
|
||||
// the catalog delete removes entry and data together.
|
||||
func TestDeleteIndexRemovesDatasetDirectory(t *testing.T) {
|
||||
|
||||
@@ -65,6 +65,10 @@ func (s *Server) SetRegion(region string) { s.region = region }
|
||||
// SetAccountID configures the account id embedded in the ARNs this server mints.
|
||||
func (s *Server) SetAccountID(id string) { s.accountID = id }
|
||||
|
||||
// SetEncryptionChecker tells CreateVectorBucket whether the gateway can
|
||||
// apply the AES256 a bucket declares.
|
||||
func (s *Server) SetEncryptionChecker(fn func() bool) { s.internal.SetEncryptionChecker(fn) }
|
||||
|
||||
// RegisterRoutes mounts POST / dispatching on the X-Amz-Target header and
|
||||
// POST /{Operation} for clients that route per-path. Metrics wrap Auth like the
|
||||
// S3 track middleware so rejected requests still count toward the operation.
|
||||
|
||||
Reference in new issue
Block a user