mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-22 22:11:39 +02:00
* s3api: regression test for UploadPartCopy ETag on ranged copies Generated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: hash the copied bytes for a partial-chunk UploadPartCopy ETag A ranged UploadPartCopy that lands inside a source chunk stored the source chunk's ETag on the destination chunk, so the part reported the MD5 of the wrong bytes. streamCopyChunkRange now tees the forwarded plaintext into an optional digest writer; copySingleChunkForRange feeds it an md5 only when the range is not the whole source chunk, since a whole-chunk copy's stored bytes (and ETag) are unchanged.\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: stamp the copied range's MD5 on UploadPartCopy part entries The part entry got no Attributes.Md5, so copyEntryETag fell back to filer.ETagChunks: a single-chunk part inherited the source chunk's MD5 and a multi-chunk part reported a <md5>-N composite of source chunks. copyChunksForRange now returns the MD5 of the copied bytes — a single chunk's own digest, or for multi-chunk parts a second ordered stream of the source range hashed beside the concurrent chunk copies (md5 cannot combine per-chunk digests). The handler stores it on Attributes.Md5 and mirrors putToFiler's ExtETagKey, so CopyPartResult, ListParts and the completed object's composite ETag use each part's own MD5 (#11855).\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: drop the source chunk's gzip flag on partial-range copies A ranged UploadPartCopy fetches raw (decompressed) bytes from the source volume and stores them uncompressed, but the destination chunk kept the source's IsCompressed flag, so reads of a partial copy of a gzip-stored chunk would try to decompress raw bytes.\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: hash the chunk range, not the entry, for the part MD5 getEncryptedStreamFromVolumes prefers inline Content over chunks, but copyChunksForRange copies the chunks — on an entry carrying both, the hash would describe different bytes than the copy.\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: cover sparse ranges in the part MD5 and fail on hash errors A single chunk's digest is the part MD5 only when it supplies the whole range; a sparse source zero-fills the rest, so gaps are hashed via the ordered range stream, and a fully chunkless range hashes zeros locally. A failed hash pass now fails the part copy instead of storing a chunk-composite ETag that describes the wrong bytes.\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: rank copied-range destination chunks by source timestamps The concurrent chunk copies stamped each destination chunk with an arbitrary finish time, but reads resolve overlapping chunks by ModifiedTsNs — a copy of overlapping source chunks could expose different bytes than the source (and than the part MD5 just hashed). Destination chunks now get timestamps ranked by the source's, preserving its overlap precedence.\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: cancel the part-MD5 stream when a chunk copy fails The hashing goroutine ran on context.Background(), so an early chunk-copy error return left it streaming and hashing the whole source range for a discarded result. A cancellable context now aborts it on every exit path.\n\nGenerated with [Devin](https://devin.ai)\n\nCo-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: hash the copied range when the covering chunk has no usable ETag TUS and mount writes leave chunks without an ETag, so a whole-range UploadPartCopy would otherwise save the part with an empty Md5 and no ExtETagKey, and multipart completion rejects the part. * s3api: build the md5 source entry without copying the proto message * s3api: drop the issue reference from the part-copy etag test * s3api: route inline-content sources through the plaintext part copy The raw-byte path walks entry.GetChunks() only, so a ranged UploadPartCopy of an inline-Content source (filer HTTP or mount writes) produced a zero-filled part. Send those through copyObjectPartViaReencryption, which streams the real bytes and stamps their MD5 on the part. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: store the empty-part MD5 on writeEmptyCopyPart entries The slow path's empty part returned the right CopyPartResult ETag but saved no Attributes.Md5 or ExtETagKey, so ListParts and the completed object's composite ETag still fell back to a chunk-derived value. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
95 lines
3.4 KiB
Go
95 lines
3.4 KiB
Go
package s3api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
|
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
|
)
|
|
|
|
// readOnlyAssignFiler answers AssignVolume the way a filer does for a path
|
|
// under a read-only rule (e.g. an over-quota bucket), and counts the calls.
|
|
type readOnlyAssignFiler struct {
|
|
filer_pb.UnimplementedSeaweedFilerServer
|
|
calls int32
|
|
}
|
|
|
|
func (f *readOnlyAssignFiler) AssignVolume(context.Context, *filer_pb.AssignVolumeRequest) (*filer_pb.AssignVolumeResponse, error) {
|
|
atomic.AddInt32(&f.calls, 1)
|
|
return &filer_pb.AssignVolumeResponse{
|
|
Error: "assign volume: read only: /buckets/b (e.g. bucket over quota)",
|
|
ErrorCode: filer_pb.FilerError_READ_ONLY,
|
|
}, nil
|
|
}
|
|
|
|
// LookupDirectoryEntry finds nothing, so the bucket has no config to apply.
|
|
func (f *readOnlyAssignFiler) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
|
return &filer_pb.LookupDirectoryEntryResponse{}, nil
|
|
}
|
|
|
|
// startReadOnlyFilers starts a read-only filer followed by a spare one, so a
|
|
// test can assert the verdict was not treated as a reason to fail over.
|
|
func startReadOnlyFilers(t *testing.T) (*S3ApiServer, *readOnlyAssignFiler, *readOnlyAssignFiler) {
|
|
t.Helper()
|
|
first, spare := &readOnlyAssignFiler{}, &readOnlyAssignFiler{}
|
|
s3a := newPutTestServer(t, startFakeFiler(t, first), startFakeFiler(t, spare))
|
|
return s3a, first, spare
|
|
}
|
|
|
|
func assertNoFailover(t *testing.T, first, spare *readOnlyAssignFiler) {
|
|
t.Helper()
|
|
if calls := atomic.LoadInt32(&first.calls); calls == 0 {
|
|
t.Fatal("the first filer was never asked")
|
|
}
|
|
if calls := atomic.LoadInt32(&spare.calls); calls != 0 {
|
|
t.Fatalf("read-only verdict failed over to the next filer %d time(s)", calls)
|
|
}
|
|
}
|
|
|
|
// PutObject into an over-quota bucket is 403, not a retryable 500.
|
|
func TestPutToFilerReadOnlyBucketIsAccessDenied(t *testing.T) {
|
|
s3a, first, spare := startReadOnlyFilers(t)
|
|
|
|
if _, code := putTestObject(t, s3a); code != s3err.ErrAccessDenied {
|
|
t.Fatalf("putToFiler = %v, want %v", code, s3err.ErrAccessDenied)
|
|
}
|
|
assertNoFailover(t, first, spare)
|
|
}
|
|
|
|
func TestAssignNewVolumeReadOnlyKeepsSentinel(t *testing.T) {
|
|
s3a, first, spare := startReadOnlyFilers(t)
|
|
|
|
_, err := s3a.assignNewVolume("/buckets/b/o", 1)
|
|
if !errors.Is(err, weed_server.ErrReadOnly) {
|
|
t.Fatalf("assignNewVolume err = %v, want ErrReadOnly", err)
|
|
}
|
|
assertNoFailover(t, first, spare)
|
|
}
|
|
|
|
// The copy paths fan chunks out to workers; the sentinel must survive the
|
|
// per-chunk wrapping so CopyObject and UploadPartCopy report 403 too.
|
|
func TestCopyChunksReadOnlyIsAccessDenied(t *testing.T) {
|
|
s3a, _, _ := startReadOnlyFilers(t)
|
|
entry := &filer_pb.Entry{
|
|
Attributes: &filer_pb.FuseAttributes{FileSize: 8},
|
|
Chunks: []*filer_pb.FileChunk{
|
|
{FileId: "3,01637037d6", Offset: 0, Size: 4},
|
|
{FileId: "3,02637037d6", Offset: 4, Size: 4},
|
|
},
|
|
}
|
|
|
|
_, err := s3a.copyChunks(entry, "/buckets/b/dst")
|
|
if code := s3a.mapCopyErrorToS3Error(err); code != s3err.ErrAccessDenied {
|
|
t.Fatalf("copyChunks err %v maps to %v, want %v", err, code, s3err.ErrAccessDenied)
|
|
}
|
|
|
|
_, _, err = s3a.copyChunksForRange(entry, 0, 7, "/buckets/b/dst")
|
|
if code := s3a.mapCopyErrorToS3Error(err); code != s3err.ErrAccessDenied {
|
|
t.Fatalf("copyChunksForRange err %v maps to %v, want %v", err, code, s3err.ErrAccessDenied)
|
|
}
|
|
}
|