mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
s3api: keep a multipart upload's state on the object's owner filer (#12668)
* s3api: create a multipart upload's directory on the object's owner filer CreateMultipartUpload wrote .uploads/<id> to the gateway's own filer while the upload's completion commits on the object's write owner. With independent filer stores, a part or completion reaching another gateway before peer metadata replication caught up could not see the upload. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: look up a multipart upload on the object's owner filer UploadPart, UploadPartCopy, ListParts, CompleteMultipartUpload, AbortMultipartUpload and the bucket policy SSE lookup read .uploads/<id> from the gateway's own filer, so a request on another gateway shortly after CreateMultipartUpload answered NoSuchUpload until replication caught up. Read it preferring the object's write owner, like object reads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: write multipart parts to the upload's owner filer UploadPart and UploadPartCopy stored part entries on the gateway's own filer, so ListParts and CompleteMultipartUpload on another gateway could miss a part that replication had not delivered yet. Store them next to the upload directory on the object's write owner. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: list and clean up a multipart upload on its owner filer ListParts, CompleteMultipartUpload and AbortMultipartUpload listed and removed part entries on the gateway's own filer. Do it where the upload and its parts now live, so completion sees every part and cleanup does not miss entries replication has not delivered yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: test a multipart upload against a lagging gateway filer Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: check the upload's owner before freeing a failed part's chunks A part write that the owner stored but answered with an error went to recovery with no object, so the owner was never among the filers checked. When the owner is outside the gateway's filer list, the others' not-found answers freed chunks the stored part still references. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: list an upload's parts on the prior owner after a ring change The upload record lookup already falls back to the key's prior owner while a ring change is in its cooling-off window, but the part listing read only the new owner. ListParts then reported no parts for an open upload. Fall back the same way when the new owner has nothing yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: find the upload record on the prior owner past a bare directory A part routed to a new owner that has not received the upload yet makes that filer create .uploads/<id> without the upload's key. The post-write check then read it as an aborted upload and removed the part. While the ring change is in its cooling-off window, a record on the prior owner now wins over such a directory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: merge the prior owner's parts into an upload listing During a ring change's cooling-off window the new owner may have received only some of an upload's parts. Listing returned the new owner's parts alone once it had any, so ListParts dropped the rest and a completion naming them failed with InvalidPart. Both owners' pages are now merged, the owner's entry winning a shared name, and cut to the page limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: fail an upload listing the prior owner could not answer When the new owner has none of an upload's parts yet and the prior owner's listing fails, the parts may all be on the prior owner. That listing read as empty, so CompleteMultipartUpload answered NoSuchUpload for an upload that still exists; it now returns the error. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: fail an upload lookup the prior owner could not answer During a ring change's cooling-off window an upload the new owner has not received yet, or holds only as a bare directory, read as NoSuchUpload when the prior owner's lookup failed. That failure is now returned, so the client sees a retryable error instead of an upload that is gone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5a1b0abc20
commit
df30c883fe
15 files changed
+568
-60
No files matched your search
@@ -0,0 +1,350 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// The gateway's own filer never receives the owner's metadata here, so every
|
||||
// multipart step after CreateMultipartUpload must find the upload and its
|
||||
// parts on the object's write owner.
|
||||
func TestMultipartUploadOnOwnerFiler(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
owner := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
replica := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
ownerAddr := startFakeFiler(t, owner)
|
||||
replicaAddr := startFakeFiler(t, replica)
|
||||
s3a := newPutTestServer(t, replicaAddr)
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, ownerAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{ownerAddr}, 1)
|
||||
|
||||
has := func(f *ambiguousPutFiler, path string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.entries[path] != nil
|
||||
}
|
||||
partsUnder := func(f *ambiguousPutFiler, dir string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
n := 0
|
||||
for key, entry := range f.entries {
|
||||
if strings.HasPrefix(key, dir+"/") && !entry.IsDirectory {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
initiate := func() string {
|
||||
r := httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil)
|
||||
out, code := s3a.createMultipartUpload(r, &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
require.True(t, has(owner, uploadDir), "the upload must be created on the owner")
|
||||
require.False(t, has(replica, uploadDir), "the gateway's filer must still be behind")
|
||||
return *out.UploadId
|
||||
}
|
||||
uploadPart := func(uploadID string, partNumber int, body string) string {
|
||||
r := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/b/dir/object?partNumber=%d&uploadId=%s", partNumber, uploadID), strings.NewReader(body))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
etag := w.Header()["ETag"]
|
||||
require.Len(t, etag, 1)
|
||||
return etag[0]
|
||||
}
|
||||
|
||||
uploadID := initiate()
|
||||
uploadDir := "/buckets/b/.uploads/" + uploadID
|
||||
etag1 := uploadPart(uploadID, 1, strings.Repeat("a", 5*1024*1024))
|
||||
etag2 := uploadPart(uploadID, 2, "tail")
|
||||
require.Equal(t, 2, partsUnder(owner, uploadDir), "parts must be stored next to the upload")
|
||||
require.Equal(t, 0, partsUnder(replica, uploadDir))
|
||||
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(uploadID),
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.Len(t, listed.Part, 2)
|
||||
|
||||
complete := httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+uploadID, nil)
|
||||
_, code = s3a.completeMultipartUpload(complete, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(uploadID),
|
||||
}, &CompleteMultipartUpload{Parts: []CompletedPart{{ETag: etag1, PartNumber: 1}, {ETag: etag2, PartNumber: 2}}})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.True(t, has(owner, "/buckets/b/dir/object"), "the object must commit on the owner")
|
||||
require.False(t, has(owner, uploadDir), "completion must remove the upload")
|
||||
|
||||
aborted := initiate()
|
||||
uploadPart(aborted, 1, "data")
|
||||
_, code = s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(aborted),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.False(t, has(owner, "/buckets/b/.uploads/"+aborted), "abort must remove the upload")
|
||||
}
|
||||
|
||||
// A part create the owner stored but answered with an error must be found
|
||||
// there before its chunks are freed, even though the owner is not one of the
|
||||
// gateway's own filers.
|
||||
func TestMultipartPartCreateErrorChecksUploadOwner(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
owner := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{},
|
||||
respError: "create parent directories: i/o timeout"}
|
||||
replica := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}
|
||||
ownerAddr := startFakeFiler(t, owner)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, replica))
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, ownerAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{ownerAddr}, 1)
|
||||
|
||||
r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId=up1", nil), s3a.uploadFiler("b", "dir/object"))
|
||||
etag, code, _ := s3a.putToFiler(r, "/buckets/b/.uploads/up1/0001_x.part", strings.NewReader("part data"), "b", "", 1, 0, nil, false, "/buckets/b/dir/object")
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.NotEmpty(t, etag)
|
||||
require.Empty(t, volume.deleted(), "chunks under the stored part must survive")
|
||||
}
|
||||
|
||||
// Right after the key moves to a new owner that has not received the upload
|
||||
// yet, ListParts finds the record and its parts on the prior owner.
|
||||
func TestListPartsAfterOwnerChange(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data"))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
require.Equal(t, priorAddr, s3a.priorWriteOwner("b", "dir/object"))
|
||||
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.Len(t, listed.Part, 1)
|
||||
}
|
||||
|
||||
// A part routed to a new owner that has not received the upload yet makes that
|
||||
// filer create a bare .uploads/<id> directory; the upload must stay open.
|
||||
func TestUploadPartAfterOwnerChange(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
next.beforeCreate = func() {
|
||||
if next.entries[uploadDir] == nil {
|
||||
next.entries[uploadDir] = &filer_pb.Entry{Name: *out.UploadId, IsDirectory: true}
|
||||
}
|
||||
}
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data"))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.Len(t, listed.Part, 1)
|
||||
}
|
||||
|
||||
// A new owner that received only some of the parts so far lists them together
|
||||
// with the rest still on the prior owner.
|
||||
func TestListPartsMergesPriorOwnerParts(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
etags := map[int]string{}
|
||||
for partNumber, body := range []string{strings.Repeat("a", 5*1024*1024), "tail"} {
|
||||
r := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/b/dir/object?partNumber=%d&uploadId=%s", partNumber+1, *out.UploadId), strings.NewReader(body))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
require.Len(t, w.Header()["ETag"], 1)
|
||||
etags[partNumber+1] = w.Header()["ETag"][0]
|
||||
}
|
||||
|
||||
prior.mu.Lock()
|
||||
for key, entry := range prior.entries {
|
||||
if key == uploadDir || strings.HasPrefix(key, uploadDir+"/0001") {
|
||||
next.entries[key] = entry
|
||||
}
|
||||
}
|
||||
prior.mu.Unlock()
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
listParts := func(maxParts, marker int64) *ListPartsResult {
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(maxParts), PartNumberMarker: aws.Int64(marker),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
return listed
|
||||
}
|
||||
require.Len(t, listParts(1000, 0).Part, 2)
|
||||
first := listParts(1, 0)
|
||||
require.Len(t, first.Part, 1)
|
||||
require.Equal(t, int64(1), *first.Part[0].PartNumber)
|
||||
require.True(t, *first.IsTruncated)
|
||||
second := listParts(1, 1)
|
||||
require.Len(t, second.Part, 1)
|
||||
require.Equal(t, int64(2), *second.Part[0].PartNumber)
|
||||
require.False(t, *second.IsTruncated)
|
||||
|
||||
_, code = s3a.completeMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+*out.UploadId, nil),
|
||||
&s3.CompleteMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId},
|
||||
&CompleteMultipartUpload{Parts: []CompletedPart{{ETag: etags[1], PartNumber: 1}, {ETag: etags[2], PartNumber: 2}}})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
}
|
||||
|
||||
type failingListFiler struct {
|
||||
*ambiguousPutFiler
|
||||
}
|
||||
|
||||
func (f failingListFiler) ListEntries(*filer_pb.ListEntriesRequest, filer_pb.SeaweedFiler_ListEntriesServer) error {
|
||||
return status.Error(codes.Unavailable, "store unavailable")
|
||||
}
|
||||
|
||||
// With no parts on the new owner yet, a prior owner that cannot be listed may
|
||||
// hold all of them, so completion must not read the upload as empty.
|
||||
func TestCompleteFailsWhenPriorOwnerListingFails(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, failingListFiler{prior})
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data"))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
require.Len(t, w.Header()["ETag"], 1)
|
||||
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
prior.mu.Lock()
|
||||
next.entries[uploadDir] = prior.entries[uploadDir]
|
||||
prior.mu.Unlock()
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
_, code = s3a.completeMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+*out.UploadId, nil),
|
||||
&s3.CompleteMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId},
|
||||
&CompleteMultipartUpload{Parts: []CompletedPart{{ETag: w.Header()["ETag"][0], PartNumber: 1}}})
|
||||
require.Equal(t, s3err.ErrInternalError, code)
|
||||
}
|
||||
|
||||
// A prior owner that cannot be asked for the upload record must not make an
|
||||
// upload the new owner has not received yet read as gone.
|
||||
func TestUploadLookupFailsWhenPriorOwnerUnreachable(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
prior.mu.Lock()
|
||||
prior.lookupErr, prior.lookupFailKey = status.Error(codes.Unavailable, "store unavailable"), uploadDir
|
||||
prior.mu.Unlock()
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
listParts := func() s3err.ErrorCode {
|
||||
_, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
return code
|
||||
}
|
||||
require.Equal(t, s3err.ErrInternalError, listParts(), "new owner has nothing yet")
|
||||
next.mu.Lock()
|
||||
next.entries[uploadDir] = &filer_pb.Entry{Name: *out.UploadId, IsDirectory: true}
|
||||
next.mu.Unlock()
|
||||
require.Equal(t, s3err.ErrInternalError, listParts(), "new owner has only a bare directory")
|
||||
}
|
||||
Reference in new issue
Block a user