mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 20:44:28 +02:00
Chris Lu
e6a0534e3b
* filer_pb: map PRECONDITION_FAILED to a sentinel error A conditional CreateEntry that the owner's path lock refuses comes back as FilerError_PRECONDITION_FAILED; mapping it to ErrPreconditionFailed lets the S3 gateway tell a rejected precondition apart from a transport failure and report 412 instead of retrying the write. * s3api: make the ambiguous-PUT recovery create conditional under the lock createAfterAmbiguousRoute verified state and then issued an unconditional create, so a write landing between the lookup and the create was still clobbered by the stale re-commit (issue #11631). The recovery create now carries a WriteCondition — IF_NOT_EXISTS for a proven-absent entry, IF_ENTRY_EQUAL for a directory it may need to write onto — which the owner filer evaluates under its per-path lock, making the check and the write atomic. A refused precondition loops back to re-read the state instead of overwriting whatever won the race. * s3api: do not fail over a routed transaction past a possibly-sent request withFilerClientFailover retried the callback on the next filer on any transport error, so an ObjectTransaction whose response was lost could be re-applied over a write that committed in between (issue #11631). A gRPC error cannot distinguish "never sent" from "committed", but channel readiness can: a non-replayable callback now runs only on a connection that just reached READY, and failover continues only over channels that never came up — the one failure mode that provably sent nothing. * s3api: run ambiguous-route recovery for empty PUTs too The recovery was gated on a nonempty chunk list, so a zero-byte PUT whose routed transaction lost its response still fell back to an unconditional create that could overwrite a newer write and delete its chunks. Chunk-count mismatch on an empty upload list is enough to distinguish a superseding entry from this PUT's — there is nothing to save by skipping the check. * s3api: keep conditional recovery creates off the replay path The conditional create in the ambiguous-PUT fallback rode the replayable WithFilerClient failover, so a committed create whose response was lost could be re-sent to another filer — where a delete in between would let IF_NOT_EXISTS pass again and resurrect an entry pointing at reclaimed chunks. Conditional creates now run through the non-replayable failover; an ambiguous result falls through to confirmCreateLanded, which resolves the committed entry rather than resending. * s3api: do not mark a filer unhealthy for an answered request A reply that reached the callback — including an application-level failure like a refused write precondition — proves the filer is alive. Checking received before the health bookkeeping keeps definitive application errors from tripping the failure circuit or flagging a live routed owner as unreachable. * s3api: do not claim an empty stored entry as an ambiguous PUT's own Two chunkless entries compare equal under sameFileChunks, so an older empty object satisfies "the route landed" even though it predates this write — recovery would return success while leaving the request's Content-Type and metadata unapplied. A landed PUT stores the entry exactly as built, so require mime/size/md5/content and the extended attributes to match before claiming it; anything else stays uncertain and the client retries. * s3api: prove ambiguous empty PUTs by write id and pin recovery to the owner - Conditional recovery creates are sent to the routed owner alone instead of failing over: the condition only serializes with routed writes under the owner's per-path lock, and another filer's forwarding is only as correct as its ring view. An unreachable owner now yields an uncertain result rather than a create under a different serialization boundary. - A proven-absent path no longer satisfies an empty, chunkless PUT: the routed commit may have landed and been deleted, and there are no chunks to verify, so IF_NOT_EXISTS could resurrect a removed object. Recovery returns uncertain for the client to retry. - Chunkless identity moves from metadata equality to a per-write id (x-seaweedfs-write-id) stamped into Extended: an identical older object can match mime/md5/content but not the id. Entries without this write's id are not claimed, and confirmCreateLanded can now positively confirm a committed empty object instead of always reporting failure.
227 lines
10 KiB
Go
227 lines
10 KiB
Go
package s3api
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"path"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
)
|
|
|
|
func ambiguousRouteServer(t *testing.T, addrs ...pb.ServerAddress) *S3ApiServer {
|
|
t.Helper()
|
|
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
|
return &S3ApiServer{
|
|
option: &S3ApiServerOption{GrpcDialOption: dialOption, Filers: addrs},
|
|
filerClient: wdclient.NewFilerClient(addrs, dialOption, ""),
|
|
}
|
|
}
|
|
|
|
func TestCreateAfterAmbiguousRoute(t *testing.T) {
|
|
uploaded := []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}
|
|
filePath := "/buckets/b/o"
|
|
chunksAlive := func(context.Context, []*filer_pb.FileChunk) bool { return true }
|
|
chunksDead := func(context.Context, []*filer_pb.FileChunk) bool { return false }
|
|
newS3a := func(t *testing.T, f *fakeLookupFiler) (*S3ApiServer, pb.ServerAddress) {
|
|
addr := startFakeFiler(t, f)
|
|
return ambiguousRouteServer(t, addr), addr
|
|
}
|
|
|
|
t.Run("stored entry is this PUT's — route landed", func(t *testing.T) {
|
|
f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}}}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrNone || !entryCreated {
|
|
t.Fatalf("code=%v entryCreated=%v", code, entryCreated)
|
|
}
|
|
if ran {
|
|
t.Fatal("re-committed an entry the route already stored")
|
|
}
|
|
})
|
|
|
|
t.Run("stored entry is a newer write — refuse stale re-commit", func(t *testing.T) {
|
|
f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d7", Size: 55}}}}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrServiceUnavailable {
|
|
t.Fatalf("code = %v, want ServiceUnavailable", code)
|
|
}
|
|
if ran || entryCreated {
|
|
t.Fatal("stale entry was committed over a newer write")
|
|
}
|
|
})
|
|
|
|
t.Run("stored entry is a directory, chunks alive — lock path answers", func(t *testing.T) {
|
|
dir := &filer_pb.Entry{Name: "o", IsDirectory: true}
|
|
f := &fakeLookupFiler{entry: dir}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
var gotCond *filer_pb.WriteCondition
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(cond *filer_pb.WriteCondition) s3err.ErrorCode {
|
|
ran = true
|
|
gotCond = cond
|
|
return s3err.ErrExistingObjectIsDirectory
|
|
})
|
|
if code != s3err.ErrExistingObjectIsDirectory || !ran {
|
|
t.Fatalf("code=%v ran=%v — a directory conflict with live chunks must reach createUnderLock, which maps it", code, ran)
|
|
}
|
|
c := oneClause(t, gotCond)
|
|
if c.Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL || !c.ExpectedEntry.IsDirectory {
|
|
t.Fatalf("directory retry carried %+v, want an IF_ENTRY_EQUAL pin on the directory", c)
|
|
}
|
|
})
|
|
|
|
// The routed PUT may have committed before a delete removed the entry and
|
|
// its chunks and a nested write recreated the name as a directory; dead
|
|
// chunks mean re-committing would store an entry pointing at them.
|
|
t.Run("stored entry is a directory, chunks unverifiable — refuse", func(t *testing.T) {
|
|
f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", IsDirectory: true}}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksDead, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrServiceUnavailable || ran {
|
|
t.Fatalf("code=%v ran=%v — re-committed over a directory with possibly-deleted chunks", code, ran)
|
|
}
|
|
})
|
|
|
|
t.Run("match behind an unanswered filer — refuse", func(t *testing.T) {
|
|
deadAddr := startFakeFiler(t, &fakeLookupFiler{lookupErr: context.DeadlineExceeded})
|
|
matchAddr := startFakeFiler(t, &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}}})
|
|
s3a := ambiguousRouteServer(t, deadAddr, matchAddr)
|
|
entryCreated := false
|
|
ran := false
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", deadAddr, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrServiceUnavailable || entryCreated || ran {
|
|
t.Fatalf("code=%v entryCreated=%v ran=%v — a match behind an unresolved lookup must not finalize", code, entryCreated, ran)
|
|
}
|
|
})
|
|
|
|
t.Run("entry proven absent, chunks alive — conditional create proceeds", func(t *testing.T) {
|
|
f := &fakeLookupFiler{}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
var gotCond *filer_pb.WriteCondition
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(cond *filer_pb.WriteCondition) s3err.ErrorCode { ran = true; gotCond = cond; return s3err.ErrNone })
|
|
if code != s3err.ErrNone || !ran {
|
|
t.Fatalf("code=%v ran=%v — proven-absent entry did not reach createUnderLock", code, ran)
|
|
}
|
|
if c := oneClause(t, gotCond); c.Kind != filer_pb.WriteCondition_IF_NOT_EXISTS {
|
|
t.Fatalf("absent-entry create carried %+v, want IF_NOT_EXISTS", c)
|
|
}
|
|
})
|
|
|
|
t.Run("entry absent but chunks unverifiable — refuse", func(t *testing.T) {
|
|
f := &fakeLookupFiler{}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksDead, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrServiceUnavailable || ran {
|
|
t.Fatalf("code=%v ran=%v — re-committed an entry over possibly-deleted chunks", code, ran)
|
|
}
|
|
})
|
|
|
|
t.Run("lookup cannot resolve state — refuse", func(t *testing.T) {
|
|
f := &fakeLookupFiler{lookupErr: context.DeadlineExceeded}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrServiceUnavailable || ran {
|
|
t.Fatalf("code=%v ran=%v — uncertain state still committed", code, ran)
|
|
}
|
|
})
|
|
|
|
t.Run("finalize fails — recovered entry rolls back", func(t *testing.T) {
|
|
f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}}}
|
|
s3a, owner := newS3a(t, f)
|
|
entryCreated := false
|
|
ran := false
|
|
finalize := &putFinalize{afterCreate: func(*filer_pb.Entry) s3err.ErrorCode { return s3err.ErrInternalError }}
|
|
code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, finalize, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
|
|
if code != s3err.ErrInternalError || ran {
|
|
t.Fatalf("code=%v ran=%v", code, ran)
|
|
}
|
|
if len(f.deleted) != 1 || f.deleted[0] != filePath {
|
|
t.Fatalf("recovered entry was not rolled back: deleted=%v", f.deleted)
|
|
}
|
|
})
|
|
}
|
|
|
|
type fakeVersionedDirFiler struct {
|
|
filer_pb.UnimplementedSeaweedFilerServer
|
|
entries map[string]*filer_pb.Entry
|
|
updates map[string]*filer_pb.Entry
|
|
}
|
|
|
|
func (f *fakeVersionedDirFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
|
if e, ok := f.entries[path.Join(req.Directory, req.Name)]; ok {
|
|
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
|
|
}
|
|
return nil, filer_pb.ErrNotFound
|
|
}
|
|
|
|
func (f *fakeVersionedDirFiler) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
|
|
f.updates[path.Join(req.Directory, req.Entry.Name)] = req.Entry
|
|
return &filer_pb.UpdateEntryResponse{}, nil
|
|
}
|
|
|
|
// A late versioned write keeps the newer latest pointer, but the version it
|
|
// just stored is born noncurrent — without a NoncurrentSinceNs stamp the
|
|
// lifecycle engine never ages it out.
|
|
func TestUpdateLatestVersionInDirectoryBornNoncurrent(t *testing.T) {
|
|
now := time.Now().UnixNano()
|
|
newerId := fmt.Sprintf("%016x%s", math.MaxInt64-now, "1111111111111111")
|
|
olderId := fmt.Sprintf("%016x%s", math.MaxInt64-(now-int64(time.Hour)), "2222222222222222")
|
|
|
|
bucketDir := "/buckets/b"
|
|
versionsPath := path.Join(bucketDir, "o"+s3_constants.VersionsFolder)
|
|
f := &fakeVersionedDirFiler{
|
|
entries: map[string]*filer_pb.Entry{
|
|
versionsPath: {
|
|
Name: "o" + s3_constants.VersionsFolder,
|
|
IsDirectory: true,
|
|
Extended: map[string][]byte{
|
|
s3_constants.ExtLatestVersionIdKey: []byte(newerId),
|
|
s3_constants.ExtLatestVersionFileNameKey: []byte("newer.v"),
|
|
},
|
|
},
|
|
path.Join(versionsPath, "late.v"): {Name: "late.v"},
|
|
},
|
|
updates: map[string]*filer_pb.Entry{},
|
|
}
|
|
s3a := ambiguousRouteServer(t, startFakeFiler(t, f))
|
|
s3a.option.BucketsPath = "/buckets"
|
|
|
|
err := s3a.updateLatestVersionInDirectory("b", "o", olderId, "late.v", &filer_pb.Entry{})
|
|
if err != nil {
|
|
t.Fatalf("updateLatestVersionInDirectory: %v", err)
|
|
}
|
|
stamped := f.updates[path.Join(versionsPath, "late.v")]
|
|
if stamped == nil {
|
|
t.Fatal("late version never got its noncurrent stamp")
|
|
}
|
|
if stamped.Extended[s3_constants.ExtNoncurrentSinceNsKey] == nil {
|
|
t.Fatal("stamp did not set ExtNoncurrentSinceNsKey")
|
|
}
|
|
if _, pointerTouched := f.updates[versionsPath]; pointerTouched {
|
|
t.Fatal("the newer latest pointer must not move")
|
|
}
|
|
}
|