mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
s3: fix UploadPartCopy with volume-data encryption (#10971)
* operation: give an encrypted chunk the plaintext ETag With -encryptVolumeData the volume server stores ciphertext, so it cannot echo a Content-MD5 back and the chunk lands with an empty ETag. Every ETag derived from those chunks then comes out empty for a single chunk, or d41d8cd98f00b204e9800998ecf8427e-N for several. The caller already hashes the plaintext to send as Content-MD5, so keep that digest as the chunk ETag instead of dropping it, and compute it for a WantMd5 caller under cipher too. * s3: re-encrypt a part copy from a volume-encrypted source UploadPartCopy raw-copies source chunks when neither side uses SSE, which also caught -encryptVolumeData sources. Those chunks are ciphertext a whole-chunk cipher key decrypts, so copying a byte range out of one and keeping the key leaves a destination that fails authentication on GET, and the copied chunks carry no ETag for the part result to report. Route them through the re-encrypting path already used for SSE: it reads the source as plaintext, hashes the part, and writes the destination under the gateway's own encryption. * s3: fetch only the range a part copy asked for The re-encrypting UploadPartCopy path opened the source at offset 0 and threw the prefix away, so assembling an object part by part read the source once per part. Now that volume-encrypted sources take this path too, that is the common case rather than an SSE corner. The chunk stream already seeks, so hand it the range. * s3: reject an unsatisfiable copy-source-range A part copy has no way to report a short part, so a range reaching past the source cannot be clamped the way a GET clamps one. The fast path silently produced a part shorter than asked for, or an empty one; the re-encrypting path pads with zeros, so a 2 MiB source copied as bytes=1048576-9999999 came back as 1 MiB of data followed by 7.5 MiB of nothing. Answer InvalidRange instead, which is what s3-tests' test_multipart_copy_invalid_range expects.
This commit is contained in:
1 parent
12fd60f92e
commit
3431bdcb74
7 files changed
+250
-26
No files matched your search
@@ -2,6 +2,8 @@ package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -173,3 +175,96 @@ func TestEncryptedVolumeCopyScenario(t *testing.T) {
|
||||
t.Log("✓ All chunk metadata properly preserved for encrypted volume copy scenario")
|
||||
})
|
||||
}
|
||||
|
||||
// A volume-encrypted source must take the re-encrypting UploadPartCopy path:
|
||||
// the raw chunk copy slices ciphertext the destination's whole-chunk cipher key
|
||||
// can no longer decrypt, and reports the part's ETag from chunks that carry
|
||||
// none (issue #10968).
|
||||
func TestSourceEntryIsEncryptedForVolumeCipher(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
entry *filer_pb.Entry
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil entry",
|
||||
entry: nil,
|
||||
},
|
||||
{
|
||||
name: "plaintext chunks",
|
||||
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
|
||||
}},
|
||||
},
|
||||
{
|
||||
name: "volume-encrypted chunk",
|
||||
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: "1,abc123", Size: 1024, CipherKey: util.GenCipherKey()},
|
||||
}},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "volume-encrypted second chunk",
|
||||
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
|
||||
{FileId: "2,def456", Offset: 1024, Size: 1024, CipherKey: util.GenCipherKey()},
|
||||
}},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "SSE-S3 chunk",
|
||||
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: "1,abc123", Size: 1024, SseType: filer_pb.SSEType_SSE_S3},
|
||||
}},
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := sourceEntryIsEncrypted(tc.entry); got != tc.want {
|
||||
t.Errorf("sourceEntryIsEncrypted = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A ranged part copy asks the chunk stream for its slice; reading the whole
|
||||
// object and discarding the prefix would make an N-part copy read the source
|
||||
// N/2 times over.
|
||||
func TestGetEncryptedStreamFromVolumesRangesInlineContent(t *testing.T) {
|
||||
s3a := &S3ApiServer{}
|
||||
entry := &filer_pb.Entry{Content: []byte("0123456789")}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
offset int64
|
||||
size int64
|
||||
want string
|
||||
}{
|
||||
{name: "whole content", size: 10, want: "0123456789"},
|
||||
{name: "leading slice", size: 4, want: "0123"},
|
||||
{name: "middle slice", offset: 3, size: 4, want: "3456"},
|
||||
{name: "trailing slice", offset: 6, size: 4, want: "6789"},
|
||||
{name: "size past the end", offset: 8, size: 10, want: "89"},
|
||||
{name: "offset past the end", offset: 10, size: 4},
|
||||
{name: "empty range", size: 0},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reader, err := s3a.getEncryptedStreamFromVolumes(context.Background(), entry, tc.offset, tc.size)
|
||||
if err != nil {
|
||||
t.Fatalf("getEncryptedStreamFromVolumes: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if string(got) != tc.want {
|
||||
t.Errorf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user