s3: evaluate multipart completion conditions under the object lock (#12209)

* filer_pb,filer: per-mutation preconditions on ObjectTransaction

A transaction's request-level condition evaluates one entry before any
mutation runs, so a commit guarding two paths — the object it publishes
and the upload directory it assembles — could not be expressed. Give each
ObjectMutation an optional condition (evaluated against condition_key,
else the mutation's target, under the same lock just before the mutation
applies) and report which mutation failed through failed_mutation on the
response, so callers can tell a mutation guard's rejection apart from the
request-level precondition's.

* s3: evaluate multipart completion conditions under the object lock

CompleteMultipartUpload checked conditional headers gateway-side before
committing, and the routed commit transaction guarded only the upload
directory — nothing made If-None-Match: * atomic with the publish, so
concurrent conditional completions of the same absent key could all
report success.

The request's reducible write condition (If-None-Match/If-Match, star or
single strong ETag) now rides the commit as the transaction's request
condition on the owner filer: on the object path for unversioned buckets,
falling back to the .versions directory for suspended and enabled
versioning. The upload-exists guard moves onto the PUT mutation, keeping
its check-before-publish ordering. Conditions the reducer cannot express
stay on the distributed write lock, where check and commit already
serialize.

* filer,s3: resolve the latest version first for versioned conditional writes

* filer: keep directory entries holding content out of the absent-slot rule

* s3: keep a gateway-side upload-exists check for filers without mutation conditions

* filer: honor null-version-is-latest resolving prefer-alt conditions

* drop scratch multipart-race repro programs

* s3: note the write lock does not cover routed writes on non-reducible conditions

weed/s3api/s3api_object_routed_write.go

@@ -216,28 +216,37 @@ func (s3a *S3ApiServer) withWriteOwnerFilerClient(owner pb.ServerAddress, fn fun
 // object path plus a RECOMPUTE_LATEST finalize, so the version's PUT and its
 // .versions pointer flip commit atomically (the recompute scans .versions/ after
 // the PUT and sees the new version).
-func (s3a *S3ApiServer) routedPut(owner pb.ServerAddress, routeKey, lockKey, filePath string, entry *filer_pb.Entry, cond *filer_pb.WriteCondition, conditionKey, conditionAltKey string, finalize []*filer_pb.ObjectMutation) (*filer_pb.ObjectTransactionResponse, error) {
+// putCondition, when set, guards the PUT mutation itself against putConditionKey
+// (e.g. the upload directory a multipart commit must still find); the request
+// condition keeps guarding the object the transaction is about.
+func (s3a *S3ApiServer) routedPut(owner pb.ServerAddress, routeKey, lockKey, filePath string, entry *filer_pb.Entry, cond *filer_pb.WriteCondition, conditionKey, conditionAltKey string, conditionPreferAlt bool, putCondition *filer_pb.WriteCondition, putConditionKey string, finalize []*filer_pb.ObjectMutation) (*filer_pb.ObjectTransactionResponse, error) {
 	mutations := make([]*filer_pb.ObjectMutation, 0, 1+len(finalize))
 	mutations = append(mutations, &filer_pb.ObjectMutation{
-		Type:      filer_pb.ObjectMutation_PUT,
-		Directory: path.Dir(filePath),
-		Entry:     entry,
+		Type:         filer_pb.ObjectMutation_PUT,
+		Directory:    path.Dir(filePath),
+		Entry:        entry,
+		Condition:    putCondition,
+		ConditionKey: putConditionKey,
 	})
 	mutations = append(mutations, finalize...)
 	return s3a.objectTxnOnFiler(owner, &filer_pb.ObjectTransactionRequest{
-		LockKey:         lockKey,
-		RouteKey:        routeKey,
-		Condition:       cond,
-		ConditionKey:    conditionKey,
-		ConditionAltKey: conditionAltKey,
-		Mutations:       mutations,
+		LockKey:            lockKey,
+		RouteKey:           routeKey,
+		Condition:          cond,
+		ConditionKey:       conditionKey,
+		ConditionAltKey:    conditionAltKey,
+		ConditionPreferAlt: conditionPreferAlt,
+		Mutations:          mutations,
 	})
 }
 
 // routedMkFile builds an entry like filer_pb.MkFile and writes it through a
 // routed PUT on the owner filer, for callers that would otherwise mkFile to the
-// default filer (e.g. multipart completion of a non-versioned object).
-func (s3a *S3ApiServer) routedMkFile(owner pb.ServerAddress, routeKey, parentDir, name string, chunks []*filer_pb.FileChunk, fn func(*filer_pb.Entry), removal *uploadRemovalTxn) error {
+// default filer (e.g. multipart completion of a non-versioned object). cond is
+// the request's write condition, evaluated on the object under the transaction
+// lock; condAltKey falls back to the .versions directory when the object slot
+// is empty (suspended or versioned buckets).
+func (s3a *S3ApiServer) routedMkFile(owner pb.ServerAddress, routeKey, parentDir, name string, chunks []*filer_pb.FileChunk, fn func(*filer_pb.Entry), removal *uploadRemovalTxn, cond *filer_pb.WriteCondition, condAltKey string) error {
 	now := time.Now().Unix()
 	entry := &filer_pb.Entry{
 		Name: name,
@@ -254,14 +263,14 @@ func (s3a *S3ApiServer) routedMkFile(owner pb.ServerAddress, routeKey, parentDir
 		fn(entry)
 	}
 	filePath := parentDir + "/" + name
-	var cond *filer_pb.WriteCondition
-	var conditionKey string
+	var putCondition *filer_pb.WriteCondition
+	var putConditionKey string
 	var finalize []*filer_pb.ObjectMutation
 	if removal != nil {
-		cond, conditionKey = removal.condition, removal.conditionKey
+		putCondition, putConditionKey = removal.putCondition, removal.putConditionKey
 		finalize = []*filer_pb.ObjectMutation{removal.mutation}
 	}
-	resp, err := s3a.routedPut(owner, routeKey, filePath, filePath, entry, cond, conditionKey, "", finalize)
+	resp, err := s3a.routedPut(owner, routeKey, filePath, filePath, entry, cond, "", condAltKey, false, putCondition, putConditionKey, finalize)
 	if err != nil {
 		return err
 	}
@@ -269,12 +278,15 @@ func (s3a *S3ApiServer) routedMkFile(owner pb.ServerAddress, routeKey, parentDir
 		// Other keys are nested under this one, so the object goes onto the
 		// directory they live in — the same retry mkFile makes off the ring.
 		entry.MarkPrefixObject()
-		resp, err = s3a.routedPut(owner, routeKey, filePath, filePath, entry, cond, conditionKey, "", finalize)
+		resp, err = s3a.routedPut(owner, routeKey, filePath, filePath, entry, cond, "", condAltKey, false, putCondition, putConditionKey, finalize)
 		if err != nil {
 			return err
 		}
 	}
 	if resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED {
+		if resp.FailedMutation == nil {
+			return errObjectPrecondition
+		}
 		return errUploadRemoved
 	}
 	if resp.Error != "" {
@@ -283,32 +295,40 @@ func (s3a *S3ApiServer) routedMkFile(owner pb.ServerAddress, routeKey, parentDir
 	return nil
 }
 
-// errUploadRemoved is a routed commit rejected because its precondition found
-// the upload directory already deleted, mapped to NoSuchUpload by callers.
+// errUploadRemoved is a routed commit rejected because the upload directory's
+// precondition found it already deleted, mapped to NoSuchUpload by callers.
 var errUploadRemoved = errors.New("upload directory already removed")
 
+// errObjectPrecondition is a routed commit rejected because the request's write
+// condition (e.g. If-None-Match: *) found the object slot already taken by a
+// racing commit, mapped to PreconditionFailed by callers.
+var errObjectPrecondition = errors.New("object precondition failed")
+
 // writeMultipartObject writes a completed multipart object entry, routed to the
 // owner when known (so it serializes with concurrent writes to the same key)
 // and falling back to a plain mkFile otherwise. removal carries the upload
 // precondition and deletion applied in the same transaction as the routed PUT.
 // routeKey must be the same key the caller used to resolve owner, so owner
-// selection and forwarding stay consistent.
-func (s3a *S3ApiServer) writeMultipartObject(owner pb.ServerAddress, routeKey, dir, name string, chunks []*filer_pb.FileChunk, fn func(*filer_pb.Entry), removal *uploadRemovalTxn) error {
+// selection and forwarding stay consistent. cond is the request's write
+// condition, enforced on the owner under the transaction lock; condAltKey is
+// the .versions directory a conditional check falls back to when the bucket
+// tracks versions.
+func (s3a *S3ApiServer) writeMultipartObject(owner pb.ServerAddress, routeKey, dir, name string, chunks []*filer_pb.FileChunk, fn func(*filer_pb.Entry), removal *uploadRemovalTxn, cond *filer_pb.WriteCondition, condAltKey string) error {
 	if owner != "" {
-		return s3a.routedMkFile(owner, routeKey, dir, name, chunks, fn, removal)
+		return s3a.routedMkFile(owner, routeKey, dir, name, chunks, fn, removal, cond, condAltKey)
 	}
 	return s3a.mkFile(dir, name, chunks, fn)
 }
 
-// uploadRemovalTxn carries the precondition and mutation that drop a completed
-// upload's directory inside the object's commit transaction. The precondition
-// fails the commit when a delete that does not take the object lock (abort,
-// lifecycle, s3.clean.uploads) removed the directory first, instead of
-// publishing the object over freed chunks.
+// uploadRemovalTxn carries the PUT precondition and mutation that drop a
+// completed upload's directory inside the object's commit transaction. The
+// precondition fails the commit when a delete that does not take the object
+// lock (abort, lifecycle, s3.clean.uploads) removed the directory first,
+// instead of publishing the object over freed chunks.
 type uploadRemovalTxn struct {
-	condition    *filer_pb.WriteCondition
-	conditionKey string
-	mutation     *filer_pb.ObjectMutation
+	putCondition    *filer_pb.WriteCondition
+	putConditionKey string
+	mutation        *filer_pb.ObjectMutation
 }
 
 // routedUploadRemoval returns the transaction parts that remove a completed
@@ -319,14 +339,20 @@ func (s3a *S3ApiServer) routedUploadRemoval(ctx context.Context, owner pb.Server
 	if owner == "" {
 		return nil, nil
 	}
+	// Older filers ignore the mutation's upload-exists condition, so during an
+	// upgrade this gateway-side check keeps the same guard window the
+	// pre-transaction code had.
+	if exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadId, true); err == nil && !exists {
+		return nil, errUploadRemoved
+	}
 	if err := s3a.deleteUnusedPartEntries(ctx, uploadDirectory, bucket, uploadId, completionState); err != nil {
 		return nil, err
 	}
 	conditionKey, condition := uploadExistsCondition(uploadDirectory)
 	return &uploadRemovalTxn{
-		condition:    condition,
-		conditionKey: conditionKey,
-		mutation:     s3a.removeUploadDirMutation(bucket, uploadId),
+		putCondition:    condition,
+		putConditionKey: conditionKey,
+		mutation:        s3a.removeUploadDirMutation(bucket, uploadId),
 	}, nil
 }