mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 12:34:29 +02:00
Chris Lu
e6a0534e3b
* filer_pb: map PRECONDITION_FAILED to a sentinel error A conditional CreateEntry that the owner's path lock refuses comes back as FilerError_PRECONDITION_FAILED; mapping it to ErrPreconditionFailed lets the S3 gateway tell a rejected precondition apart from a transport failure and report 412 instead of retrying the write. * s3api: make the ambiguous-PUT recovery create conditional under the lock createAfterAmbiguousRoute verified state and then issued an unconditional create, so a write landing between the lookup and the create was still clobbered by the stale re-commit (issue #11631). The recovery create now carries a WriteCondition — IF_NOT_EXISTS for a proven-absent entry, IF_ENTRY_EQUAL for a directory it may need to write onto — which the owner filer evaluates under its per-path lock, making the check and the write atomic. A refused precondition loops back to re-read the state instead of overwriting whatever won the race. * s3api: do not fail over a routed transaction past a possibly-sent request withFilerClientFailover retried the callback on the next filer on any transport error, so an ObjectTransaction whose response was lost could be re-applied over a write that committed in between (issue #11631). A gRPC error cannot distinguish "never sent" from "committed", but channel readiness can: a non-replayable callback now runs only on a connection that just reached READY, and failover continues only over channels that never came up — the one failure mode that provably sent nothing. * s3api: run ambiguous-route recovery for empty PUTs too The recovery was gated on a nonempty chunk list, so a zero-byte PUT whose routed transaction lost its response still fell back to an unconditional create that could overwrite a newer write and delete its chunks. Chunk-count mismatch on an empty upload list is enough to distinguish a superseding entry from this PUT's — there is nothing to save by skipping the check. * s3api: keep conditional recovery creates off the replay path The conditional create in the ambiguous-PUT fallback rode the replayable WithFilerClient failover, so a committed create whose response was lost could be re-sent to another filer — where a delete in between would let IF_NOT_EXISTS pass again and resurrect an entry pointing at reclaimed chunks. Conditional creates now run through the non-replayable failover; an ambiguous result falls through to confirmCreateLanded, which resolves the committed entry rather than resending. * s3api: do not mark a filer unhealthy for an answered request A reply that reached the callback — including an application-level failure like a refused write precondition — proves the filer is alive. Checking received before the health bookkeeping keeps definitive application errors from tripping the failure circuit or flagging a live routed owner as unreachable. * s3api: do not claim an empty stored entry as an ambiguous PUT's own Two chunkless entries compare equal under sameFileChunks, so an older empty object satisfies "the route landed" even though it predates this write — recovery would return success while leaving the request's Content-Type and metadata unapplied. A landed PUT stores the entry exactly as built, so require mime/size/md5/content and the extended attributes to match before claiming it; anything else stays uncertain and the client retries. * s3api: prove ambiguous empty PUTs by write id and pin recovery to the owner - Conditional recovery creates are sent to the routed owner alone instead of failing over: the condition only serializes with routed writes under the owner's per-path lock, and another filer's forwarding is only as correct as its ring view. An unreachable owner now yields an uncertain result rather than a create under a different serialization boundary. - A proven-absent path no longer satisfies an empty, chunkless PUT: the routed commit may have landed and been deleted, and there are no chunks to verify, so IF_NOT_EXISTS could resurrect a removed object. Recovery returns uncertain for the client to retry. - Chunkless identity moves from metadata equality to a per-write id (x-seaweedfs-write-id) stamped into Extended: an identical older object can match mime/md5/content but not the id. Entries without this write's id are not claimed, and confirmCreateLanded can now positively confirm a committed empty object instead of always reporting failure.
109 lines
4.0 KiB
Go
109 lines
4.0 KiB
Go
package s3api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
// unreachableOwnerTTL is how long a route-by-key owner that just failed a read is
|
|
// skipped before being retried — long enough to spare a dead owner per-request
|
|
// dials, short enough to resume owner-first reads soon after it (or the ring) recovers.
|
|
const unreachableOwnerTTL = 2 * time.Second
|
|
|
|
// getObjectEntryRoutedByKey resolves an object's entry preferring the key's write
|
|
// owner (the same route key the write path hashes), so a read sees a just-written
|
|
// object without waiting for cross-filer replication. On the owner's ErrNotFound it
|
|
// probes the key's prior owner once during a rebalance window; falls back to
|
|
// getEntry when no owner is resolvable.
|
|
func (s3a *S3ApiServer) getObjectEntryRoutedByKey(bucket, object string) (*filer_pb.Entry, error) {
|
|
fullPath := util.NewFullPath(s3a.bucketDir(bucket), object)
|
|
|
|
owner := s3a.routableWriteOwner(bucket, object)
|
|
if owner == "" || s3a.filerClient == nil {
|
|
entry, _, _, err := filer_pb.GetEntry(context.Background(), s3a, fullPath)
|
|
return entry, err
|
|
}
|
|
|
|
// Skip an owner whose recent read hit a transport error; read local-first until
|
|
// it (or the ring) recovers, rather than re-dialing a dead owner every request.
|
|
preferred := owner
|
|
if s3a.ownerRecentlyUnreachable(owner) {
|
|
preferred = ""
|
|
}
|
|
|
|
dir, name := fullPath.DirAndName()
|
|
var entry *filer_pb.Entry
|
|
err := s3a.withFilerClientFailover(context.Background(), preferred, false, true, func(client filer_pb.SeaweedFilerClient) error {
|
|
resp, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
|
|
Directory: dir,
|
|
Name: name,
|
|
})
|
|
if lookupErr != nil {
|
|
return lookupErr
|
|
}
|
|
entry = resp.Entry
|
|
return nil
|
|
})
|
|
|
|
// A just-moved key may not have replicated to the new owner yet; consult its
|
|
// prior owner once while the ring change is within the cooling-off window.
|
|
if errors.Is(err, filer_pb.ErrNotFound) {
|
|
if prior := s3a.priorWriteOwner(bucket, object); prior != "" && prior != owner {
|
|
if priorEntry, priorErr := s3a.lookupEntryOnFiler(context.Background(), prior, dir, name); priorErr == nil {
|
|
return priorEntry, nil
|
|
}
|
|
}
|
|
}
|
|
return entry, err
|
|
}
|
|
|
|
func (s3a *S3ApiServer) priorWriteOwner(bucket, object string) pb.ServerAddress {
|
|
if object == "" || s3a.objectWriteLockClient == nil {
|
|
return ""
|
|
}
|
|
return s3a.objectWriteLockClient.PriorOwnerForKey(s3a.objectRouteKey(bucket, object))
|
|
}
|
|
|
|
func (s3a *S3ApiServer) markOwnerUnreachable(owner pb.ServerAddress) {
|
|
s3a.unreachableOwners.Store(owner, time.Now().Add(unreachableOwnerTTL))
|
|
}
|
|
|
|
func (s3a *S3ApiServer) ownerRecentlyUnreachable(owner pb.ServerAddress) bool {
|
|
if v, ok := s3a.unreachableOwners.Load(owner); ok {
|
|
return time.Now().Before(v.(time.Time))
|
|
}
|
|
return false
|
|
}
|
|
|
|
// lookupEntryPreferringOwner reads an entry back from a known write owner, so a
|
|
// caller that just wrote there sees its own write. Unlike getObjectEntryRoutedByKey
|
|
// it never drops the owner for a healthier peer, which would read behind the write.
|
|
func (s3a *S3ApiServer) lookupEntryPreferringOwner(owner pb.ServerAddress, dir, name string) (*filer_pb.Entry, error) {
|
|
if owner == "" {
|
|
return s3a.getEntry(dir, name)
|
|
}
|
|
return s3a.lookupEntryOnFiler(context.Background(), owner, dir, name)
|
|
}
|
|
|
|
// lookupEntryOnFiler resolves dir/name against a single filer, without failover.
|
|
func (s3a *S3ApiServer) lookupEntryOnFiler(ctx context.Context, filer pb.ServerAddress, dir, name string) (*filer_pb.Entry, error) {
|
|
var entry *filer_pb.Entry
|
|
err := pb.WithFilerClient(false, 0, filer, s3a.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
|
resp, lookupErr := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
|
Directory: dir,
|
|
Name: name,
|
|
})
|
|
if lookupErr != nil {
|
|
return lookupErr
|
|
}
|
|
entry = resp.Entry
|
|
return nil
|
|
})
|
|
return entry, err
|
|
}
|