s3: fix UploadPartCopy with volume-data encryption (#10971)

* operation: give an encrypted chunk the plaintext ETag

With -encryptVolumeData the volume server stores ciphertext, so it cannot
echo a Content-MD5 back and the chunk lands with an empty ETag. Every ETag
derived from those chunks then comes out empty for a single chunk, or
d41d8cd98f00b204e9800998ecf8427e-N for several.

The caller already hashes the plaintext to send as Content-MD5, so keep that
digest as the chunk ETag instead of dropping it, and compute it for a
WantMd5 caller under cipher too.

* s3: re-encrypt a part copy from a volume-encrypted source

UploadPartCopy raw-copies source chunks when neither side uses SSE, which
also caught -encryptVolumeData sources. Those chunks are ciphertext a
whole-chunk cipher key decrypts, so copying a byte range out of one and
keeping the key leaves a destination that fails authentication on GET, and
the copied chunks carry no ETag for the part result to report.

Route them through the re-encrypting path already used for SSE: it reads the
source as plaintext, hashes the part, and writes the destination under the
gateway's own encryption.

* s3: fetch only the range a part copy asked for

The re-encrypting UploadPartCopy path opened the source at offset 0 and threw
the prefix away, so assembling an object part by part read the source once per
part. Now that volume-encrypted sources take this path too, that is the common
case rather than an SSE corner.

The chunk stream already seeks, so hand it the range.

* s3: reject an unsatisfiable copy-source-range

A part copy has no way to report a short part, so a range reaching past the
source cannot be clamped the way a GET clamps one. The fast path silently
produced a part shorter than asked for, or an empty one; the re-encrypting
path pads with zeros, so a 2 MiB source copied as bytes=1048576-9999999 came
back as 1 MiB of data followed by 7.5 MiB of nothing.

Answer InvalidRange instead, which is what s3-tests'
test_multipart_copy_invalid_range expects.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-08-26 10:05:49 -07:00
1 parent 12fd60f92e
commit 3431bdcb74
7 files changed
+250 -26

No files matched your search

@@ -2,6 +2,8 @@ package s3api
import (
"bytes"
"context"
"io"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -173,3 +175,96 @@ func TestEncryptedVolumeCopyScenario(t *testing.T) {
t.Log("✓ All chunk metadata properly preserved for encrypted volume copy scenario")
})
}
// A volume-encrypted source must take the re-encrypting UploadPartCopy path:
// the raw chunk copy slices ciphertext the destination's whole-chunk cipher key
// can no longer decrypt, and reports the part's ETag from chunks that carry
// none (issue #10968).
func TestSourceEntryIsEncryptedForVolumeCipher(t *testing.T) {
testCases := []struct {
name string
entry *filer_pb.Entry
want bool
}{
{
name: "nil entry",
entry: nil,
},
{
name: "plaintext chunks",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
}},
},
{
name: "volume-encrypted chunk",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, CipherKey: util.GenCipherKey()},
}},
want: true,
},
{
name: "volume-encrypted second chunk",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
{FileId: "2,def456", Offset: 1024, Size: 1024, CipherKey: util.GenCipherKey()},
}},
want: true,
},
{
name: "SSE-S3 chunk",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, SseType: filer_pb.SSEType_SSE_S3},
}},
want: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if got := sourceEntryIsEncrypted(tc.entry); got != tc.want {
t.Errorf("sourceEntryIsEncrypted = %v, want %v", got, tc.want)
}
})
}
}
// A ranged part copy asks the chunk stream for its slice; reading the whole
// object and discarding the prefix would make an N-part copy read the source
// N/2 times over.
func TestGetEncryptedStreamFromVolumesRangesInlineContent(t *testing.T) {
s3a := &S3ApiServer{}
entry := &filer_pb.Entry{Content: []byte("0123456789")}
testCases := []struct {
name string
offset int64
size int64
want string
}{
{name: "whole content", size: 10, want: "0123456789"},
{name: "leading slice", size: 4, want: "0123"},
{name: "middle slice", offset: 3, size: 4, want: "3456"},
{name: "trailing slice", offset: 6, size: 4, want: "6789"},
{name: "size past the end", offset: 8, size: 10, want: "89"},
{name: "offset past the end", offset: 10, size: 4},
{name: "empty range", size: 0},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
reader, err := s3a.getEncryptedStreamFromVolumes(context.Background(), entry, tc.offset, tc.size)
if err != nil {
t.Fatalf("getEncryptedStreamFromVolumes: %v", err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("read: %v", err)
}
if string(got) != tc.want {
t.Errorf("got %q, want %q", got, tc.want)
}
})
}
}