fix(iceberg): keep a losing table create from deleting the winner's metadata (#12474)

* iceberg: keep a losing table create from deleting the winner's metadata

* iceberg: keep a staged file whose published record was renamed away

A create answers a conflict when its record was already visible and
another request changed it before the create could back it out. The
cleanup then looked only at the original name, so a record renamed in
between read as absent and the file it still points at was deleted.

S3 Tables now reports that case with a conflict of its own, and both
create paths keep the staged file on it instead of probing the name.
A conflict over a still-hidden record, lost to another create, still
removes the file.

* iceberg: discard a losing create's metadata only while it is still its own

The cleanup deleted the staged file by name. The path can be reused
before it runs: the table is dropped and purged, and another create
stages the same v1.metadata.json and registers it, which the losing
request then deleted.

Staged files now carry a per-request token in an internal extended
attribute, and the cleanup deletes with that token as the filer's
ExpectedExtended precondition, checked under the path lock the delete
runs under. A file that took the path since is left in place.

* iceberg: classify a typed S3 Tables error by its code, not its message

isS3TablesAlreadyExists matched "already exists" in the message before
looking at the code, so an InternalError whose text mentioned an
existing entry read as the name being taken: CreateTable answered 409
instead of 500, and registrationRefused discarded a staged file the
catalog may still point at. A typed error now decides by its code; only
an untyped one is matched by message.

* iceberg: pin stage-create on in the stage-create conflict test

Stage-create is on by default, but a shell that sets
ICEBERG_ENABLE_STAGE_CREATE=false turned the expected 409 into a 501.
Setting it in the test keeps the result independent of the environment.

* iceberg: test two creates overlapping at the canonical metadata name

The concurrent-creators test only lines the creators up before they
start, so whether two of them ever collide at v1.metadata.json is up to
the scheduler. The new test fixes the order: the second create runs
while the first has staged v1.metadata.json and not yet registered it.
It checks that every staging attempt is exclusive, that the second falls
back to v1-<uuid>, that both files hold their own metadata while both
are staged, and that the loser removes only its own file.

* s3tables: a create whose unmark write never went out stayed hidden

confirmChild counted any failure to take a create's mark off as possibly
published, the lookup that precedes the write included. That lookup
failing leaves the mark on, so the record stayed hidden; if another
create then adopted it, the back-out reported a published record that
changed, and the Iceberg catalog kept a staged file no record names.

The mark-removal calls now go through a wrapper that notes whether an
UpdateEntry went out without the filer refusing it. Only then may the
record have been seen. A refused compare, a missing entry, or a failure
before any write leaves the create's record hidden.

* s3tables: pass mayBeVisible into backOutChild

Fixes the build after rebasing onto the backOutChild split.

weed/s3api/iceberg/commit_helpers.go

@@ -35,18 +35,66 @@ func sleepBeforeCommitRetry(attempt int) {
 // decides which commit won. This mirrors how the maintenance worker stages its
 // own metadata.
 func (s *Server) stageCommitMetadata(ctx context.Context, metadataBucket, metadataPath, location, metadataFileName string, metadataBytes []byte) (string, string, error) {
-	err := s.saveMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName, metadataBytes, true)
+	name, metadataLocation, _, err := s.stageOwnedMetadata(ctx, metadataBucket, metadataPath, location, metadataFileName, metadataBytes)
+	return name, metadataLocation, err
+}
+
+// stageOwnedMetadata is stageCommitMetadata that also returns the token the
+// file was tagged with, which discardStagedMetadata needs to remove it.
+func (s *Server) stageOwnedMetadata(ctx context.Context, metadataBucket, metadataPath, location, metadataFileName string, metadataBytes []byte) (string, string, string, error) {
+	token := uuid.NewString()
+	extended := map[string][]byte{stageTokenKey: []byte(token)}
+	err := s.saveMetadataBlob(ctx, metadataBucket, metadataPath, metadataFileName, metadataBytes, "application/json", true, extended)
 	if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
 		// v{N}-{uuid}, a form both the catalog and the maintenance worker
 		// still read the version from.
 		metadataFileName = fmt.Sprintf("%s-%s.metadata.json", strings.TrimSuffix(metadataFileName, ".metadata.json"), uuid.NewString())
 		glog.V(1).Infof("Iceberg: metadata file already staged for %s, using %s", location, metadataFileName)
-		err = s.saveMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName, metadataBytes, true)
+		err = s.saveMetadataBlob(ctx, metadataBucket, metadataPath, metadataFileName, metadataBytes, "application/json", true, extended)
 	}
 	if err != nil {
-		return "", "", err
+		return "", "", "", err
 	}
-	return metadataFileName, fmt.Sprintf("%s/metadata/%s", strings.TrimSuffix(location, "/"), metadataFileName), nil
+	return metadataFileName, fmt.Sprintf("%s/metadata/%s", strings.TrimSuffix(location, "/"), metadataFileName), token, nil
+}
+
+// discardStagedMetadata removes a metadata file this request staged and the
+// catalog will never point at. metadataFileName and token must be what
+// stageOwnedMetadata returned. The name alone does not identify the file: the
+// path can be reused once this request's file is gone — the table dropped and
+// its directory purged, then a new create staging the same v1.metadata.json —
+// so the delete goes through only while the entry still carries this
+// request's token, and a replacement is left alone.
+func (s *Server) discardStagedMetadata(ctx context.Context, metadataBucket, metadataPath, metadataFileName, metadataLocation, token string) {
+	err := s.deleteStagedMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName, token)
+	if errors.Is(err, errStagedFileReplaced) {
+		glog.V(1).Infof("Iceberg: keeping metadata file %s, it is no longer the one this request staged: %v", metadataLocation, err)
+		return
+	}
+	if err != nil {
+		glog.V(1).Infof("Iceberg: failed to cleanup metadata file %s after the catalog refused it: %v", metadataLocation, err)
+	}
+}
+
+// registrationRefused reports whether a failed registration was refused
+// before any record pointing at the staged file became visible, so the file can
+// go. A name already taken, a conflict, and any other 4xx are refused that way:
+// the create wrote nothing, or backed out what it wrote, or lost its
+// still-hidden record to another create. One conflict is not: a create whose
+// record was already visible, and was updated or renamed away before the
+// create could back it out, leaves that record in place — under the original
+// name or another — still pointing at the file. A transport failure or 5xx is
+// not a refusal either: the record may have been written before the error
+// surfaced.
+func registrationRefused(err error) bool {
+	if s3tables.IsPublishedTableChanged(err) {
+		return false
+	}
+	if isS3TablesConflict(err) || isS3TablesAlreadyExists(err) {
+		return true
+	}
+	status, _, _ := classifyManagerError(err)
+	return status >= 400 && status < 500
 }
 
 type icebergRequestError struct {
@@ -102,16 +150,20 @@ func hasAssertCreateRequirement(requirements table.Requirements) bool {
 	return false
 }
 
+// isS3TablesAlreadyExists reports whether err says the name is already taken.
+// A typed error says so through its code alone: a server fault whose message
+// happens to mention an existing entry is still a server fault, and taking it
+// for a refusal would answer 409 and discard a file the catalog may point at.
+// Only an untyped error is read by its message.
 func isS3TablesAlreadyExists(err error) bool {
 	if err == nil {
 		return false
 	}
-	if strings.Contains(strings.ToLower(err.Error()), "already exists") {
-		return true
-	}
 	var tableErr *s3tables.S3TablesError
-	return errors.As(err, &tableErr) &&
-		(tableErr.Type == s3tables.ErrCodeTableAlreadyExists || tableErr.Type == s3tables.ErrCodeNamespaceAlreadyExists || strings.Contains(strings.ToLower(tableErr.Message), "already exists"))
+	if errors.As(err, &tableErr) {
+		return tableErr.Type == s3tables.ErrCodeTableAlreadyExists || tableErr.Type == s3tables.ErrCodeNamespaceAlreadyExists
+	}
+	return strings.Contains(strings.ToLower(err.Error()), "already exists")
 }
 
 func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommitInput) (*CommitTableResponse, *icebergRequestError) {
@@ -147,7 +199,6 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
 		metadataVersion = 1
 	}
 	metadataFileName := fmt.Sprintf("v%d.metadata.json", metadataVersion)
-	newMetadataLocation := fmt.Sprintf("%s/metadata/%s", strings.TrimSuffix(input.location, "/"), metadataFileName)
 
 	metadataBytes, err := json.Marshal(newMetadata)
 	if err != nil {
@@ -194,7 +245,11 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
 			message: err.Error(),
 		}
 	}
-	if err := s.saveMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName, metadataBytes, false); err != nil {
+	// Two creators racing for the same table both pick this name. Staging it
+	// exclusively gives each its own file, so the loser can discard its file
+	// below without deleting the one the winner's record points at.
+	metadataFileName, newMetadataLocation, stageToken, err := s.stageOwnedMetadata(ctx, metadataBucket, metadataPath, input.location, metadataFileName, metadataBytes)
+	if err != nil {
 		return nil, &icebergRequestError{
 			status:  http.StatusInternalServerError,
 			errType: "InternalServerError",
@@ -216,20 +271,22 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
 		MetadataVersion:  metadataVersion,
 		MetadataLocation: newMetadataLocation,
 	}
+	var createResp s3tables.CreateTableResponse
 	createErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		mgrClient := s3tables.NewManagerClient(client)
-		return s.tablesManager.Execute(ctx, mgrClient, "CreateTable", createReq, nil, input.identityName)
+		return s.tablesManager.Execute(ctx, mgrClient, "CreateTable", createReq, &createResp, input.identityName)
 	})
+	tableCreatedConcurrently := &icebergRequestError{
+		status:  http.StatusConflict,
+		errType: "CommitFailedException",
+		message: "Table was created concurrently",
+	}
 	if createErr != nil {
-		if cleanupErr := s.deleteMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName); cleanupErr != nil {
-			glog.V(1).Infof("Iceberg: failed to cleanup metadata file %s after create-on-commit failure: %v", newMetadataLocation, cleanupErr)
+		if registrationRefused(createErr) {
+			s.discardStagedMetadata(ctx, metadataBucket, metadataPath, metadataFileName, newMetadataLocation, stageToken)
 		}
 		if isS3TablesConflict(createErr) || isS3TablesAlreadyExists(createErr) {
-			return nil, &icebergRequestError{
-				status:  http.StatusConflict,
-				errType: "CommitFailedException",
-				message: "Table was created concurrently",
-			}
+			return nil, tableCreatedConcurrently
 		}
 		glog.Errorf("Iceberg: CommitTable CreateTable error: %v", createErr)
 		return nil, &icebergRequestError{
@@ -238,6 +295,13 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
 			message: "Failed to commit table creation: " + createErr.Error(),
 		}
 	}
+	// A name that already holds a table is answered with that table's own
+	// location rather than an error: another creator won the race. That
+	// location is empty for a table registered without one.
+	if createResp.MetadataLocation != newMetadataLocation {
+		s.discardStagedMetadata(ctx, metadataBucket, metadataPath, metadataFileName, newMetadataLocation, stageToken)
+		return nil, tableCreatedConcurrently
+	}
 
 	markerBucket := input.markerBucket
 	if markerBucket == "" {

weed/s3api/iceberg/handlers_table.go

@@ -269,11 +269,12 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Authoritative existence check: ask the catalog whether a table is registered
-	// at this name. If it is, short-circuit with the existing table (idempotent
-	// CreateTable). Any leftover objects at the target path from a previous
-	// lifecycle of the same name are purged by handleDropTable on the way out —
-	// we deliberately do not clean storage here to keep CreateTable free of
-	// destructive side effects. See issue #9074.
+	// at this name. If it is, refuse with 409 as the Iceberg REST spec requires;
+	// clients that want create-if-not-exists catch the 409 and load the table.
+	// Any leftover objects at the target path from a previous lifecycle of the
+	// same name are purged by handleDropTable on the way out — we deliberately do
+	// not clean storage here to keep CreateTable free of destructive side
+	// effects. See issue #9074.
 	existsReq := &s3tables.GetTableRequest{
 		TableBucketARN: bucketARN,
 		Namespace:      namespace,
@@ -285,15 +286,7 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", existsReq, &existsResp, identityName)
 	})
 	if existsErr == nil {
-		// Table already registered. Return the existing definition so CTAS/IF NOT
-		// EXISTS flows see a stable response instead of a 409.
-		result, buildErr := s.buildLoadTableResult(r, existsResp, bucketName, namespace, tableName)
-		if buildErr != nil {
-			glog.Errorf("Iceberg: CreateTable load existing %s: %v", tableName, buildErr)
-			writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to build table metadata")
-			return
-		}
-		writeLoadResult(w, http.StatusOK, result)
+		writeTableAlreadyExists(w, namespace, tableName)
 		return
 	}
 	if !isNoSuchTableError(existsErr) {
@@ -319,7 +312,9 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 	// Stage-create persists metadata in the internal staged area and skips S3Tables registration.
 	if req.StageCreate {
 		stagedTablePath := stageCreateStagedTablePath(namespace, tableName, tableUUID)
-		if err := s.saveMetadataFile(r.Context(), metadataBucket, stagedTablePath, metadataFileName, metadataBytes, false); err != nil {
+		// The staged path carries the table UUID, so the name cannot already be
+		// taken; exclusive keeps it that way if it ever is.
+		if err := s.saveMetadataFile(r.Context(), metadataBucket, stagedTablePath, metadataFileName, metadataBytes, true); err != nil {
 			writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save staged metadata file: "+err.Error())
 			return
 		}
@@ -340,7 +335,12 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		writeLoadResult(w, http.StatusOK, result)
 		return
 	}
-	if err := s.saveMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataBytes, false); err != nil {
+	// The location may already hold a v1.metadata.json: a table created there
+	// earlier and since dropped, an unregistered file, or a request racing this
+	// one for the same name. Writing it exclusively keeps this request from
+	// replacing that file, and from deleting it below if registration fails.
+	metadataFileName, metadataLocation, stageToken, err := s.stageOwnedMetadata(r.Context(), metadataBucket, metadataPath, location, metadataFileName, metadataBytes)
+	if err != nil {
 		writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save metadata file: "+err.Error())
 		return
 	}
@@ -368,53 +368,17 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		var tableErr *s3tables.S3TablesError
-		if errors.As(err, &tableErr) && tableErr.Type == s3tables.ErrCodeTableAlreadyExists {
-			getReq := &s3tables.GetTableRequest{
-				TableBucketARN: bucketARN,
-				Namespace:      namespace,
-				Name:           tableName,
-			}
-			var getResp s3tables.GetTableResponse
-			getErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-				mgrClient := s3tables.NewManagerClient(client)
-				return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
-			})
-			if getErr != nil {
-				writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
-				return
-			}
-			result, buildErr := s.buildLoadTableResult(r, getResp, bucketName, namespace, tableName)
-			if buildErr != nil {
-				glog.Errorf("Iceberg: CreateTable load existing %s: %v", tableName, buildErr)
-				writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to build table metadata")
-				return
-			}
-			writeLoadResult(w, http.StatusOK, result)
-			return
+		// The file is named by this request alone, so it can go once the
+		// catalog refused the registration before a record naming it became
+		// visible. After a failure that may have left one (a conflict on a
+		// record already published, transport, 5xx) it stays: the record,
+		// under this name or one it was renamed to, could point at it, and
+		// removing it would leave the table unreadable.
+		if registrationRefused(err) {
+			s.discardStagedMetadata(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataLocation, stageToken)
 		}
-		if strings.Contains(err.Error(), "already exists") {
-			getReq := &s3tables.GetTableRequest{
-				TableBucketARN: bucketARN,
-				Namespace:      namespace,
-				Name:           tableName,
-			}
-			var getResp s3tables.GetTableResponse
-			getErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-				mgrClient := s3tables.NewManagerClient(client)
-				return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
-			})
-			if getErr != nil {
-				writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
-				return
-			}
-			result, buildErr := s.buildLoadTableResult(r, getResp, bucketName, namespace, tableName)
-			if buildErr != nil {
-				glog.Errorf("Iceberg: CreateTable load existing %s: %v", tableName, buildErr)
-				writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to build table metadata")
-				return
-			}
-			writeLoadResult(w, http.StatusOK, result)
+		if isS3TablesAlreadyExists(err) {
+			writeTableAlreadyExists(w, namespace, tableName)
 			return
 		}
 		glog.V(1).Infof("Iceberg: CreateTable error: %v", err)
@@ -422,11 +386,17 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Use returned location if available, otherwise fallback to local one
-	finalLocation := createResp.MetadataLocation
-	if finalLocation == "" {
-		finalLocation = metadataLocation
+	// The catalog answers a create for a name that already holds a table with
+	// that table's own metadata location rather than an error. The table was
+	// registered by a request that won the race, not by this one. A winner
+	// registered without a location comes back empty, which is no match either:
+	// this request always sends one, and a fresh create echoes it.
+	if createResp.MetadataLocation != metadataLocation {
+		s.discardStagedMetadata(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataLocation, stageToken)
+		writeTableAlreadyExists(w, namespace, tableName)
+		return
 	}
+	finalLocation := metadataLocation
 	if markerErr := s.deleteStageCreateMarkers(r.Context(), bucketName, namespace, tableName); markerErr != nil {
 		glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after create: %v", flattenNamespacePath(namespace), tableName, markerErr)
 	}
@@ -993,6 +963,12 @@ func (s *Server) handleRenameTable(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+// writeTableAlreadyExists answers a create for a name that holds a table with
+// the 409 the Iceberg REST spec defines for it.
+func writeTableAlreadyExists(w http.ResponseWriter, namespace []string, tableName string) {
+	writeError(w, http.StatusConflict, "AlreadyExistsException", fmt.Sprintf("Table already exists: %s.%s", strings.Join(namespace, "."), tableName))
+}
+
 // isNoSuchTableError reports whether an error from the S3 Tables manager
 // indicates the target table is not registered in the catalog.
 func isNoSuchTableError(err error) bool {

weed/s3api/iceberg/manifest_repair.go

@@ -556,7 +556,7 @@ func (st *serverManifestStore) saveFile(ctx context.Context, location string, da
 	if err != nil {
 		return err
 	}
-	return st.server.saveMetadataBlob(ctx, bucket, tablePath, fileName, data, "application/avro", false)
+	return st.server.saveMetadataBlob(ctx, bucket, tablePath, fileName, data, "application/avro", false, nil)
 }
 
 // repairAddSnapshotManifests is the server entry point used by the commit

weed/s3api/iceberg/metadata_files.go

@@ -10,9 +10,18 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
+// stageTokenKey is the extended attribute stageOwnedMetadata tags a staged
+// metadata file with: a value unique to the request that wrote it, which the
+// request's own cleanup must find before it deletes the file. The internal
+// prefix keeps it out of the headers S3 reads return.
+const stageTokenKey = s3_constants.SeaweedFSInternalPrefix + "iceberg-stage-token"
+
 // metadataDirPath returns the filer directory holding a table's metadata files.
 func metadataDirPath(bucketName, tablePath string) string {
 	dir := path.Join(s3tables.TablesPath, bucketName)
@@ -29,11 +38,16 @@ func metadataDirPath(bucketName, tablePath string) string {
 // same v{N} name, and letting the second overwrite the first would leave the
 // winner's catalog pointer aimed at the loser's metadata.
 func (s *Server) saveMetadataFile(ctx context.Context, bucketName, tablePath, metadataFileName string, content []byte, exclusive bool) error {
-	return s.saveMetadataBlob(ctx, bucketName, tablePath, metadataFileName, content, "application/json", exclusive)
+	return s.saveMetadataBlob(ctx, bucketName, tablePath, metadataFileName, content, "application/json", exclusive, nil)
 }
 
-// saveMetadataBlob saves a file into the table's metadata directory.
-func (s *Server) saveMetadataBlob(ctx context.Context, bucketName, tablePath, metadataFileName string, content []byte, mimeType string, exclusive bool) error {
+// saveMetadataBlob saves a file into the table's metadata directory, with
+// extended stored on the entry alongside its mime type.
+func (s *Server) saveMetadataBlob(ctx context.Context, bucketName, tablePath, metadataFileName string, content []byte, mimeType string, exclusive bool, extended map[string][]byte) error {
+	attrs := map[string][]byte{"Mime-Type": []byte(mimeType)}
+	for key, value := range extended {
+		attrs[key] = value
+	}
 
 	// Create context with timeout for file operations
 	opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
@@ -115,10 +129,8 @@ func (s *Server) saveMetadataBlob(ctx context.Context, bucketName, tablePath, me
 					FileMode: uint32(0644),
 					FileSize: uint64(len(content)),
 				},
-				Content: content,
-				Extended: map[string][]byte{
-					"Mime-Type": []byte(mimeType),
-				},
+				Content:  content,
+				Extended: attrs,
 			},
 		})
 		if err != nil {
@@ -144,6 +156,39 @@ func (s *Server) deleteMetadataFile(ctx context.Context, bucketName, tablePath,
 	})
 }
 
+// errStagedFileReplaced reports that the entry at a staged file's path no
+// longer carries the token it was staged with: it is another writer's file.
+var errStagedFileReplaced = errors.New("staged metadata file was replaced")
+
+// deleteStagedMetadataFile removes a file stageOwnedMetadata wrote, only while
+// the entry at its path still carries token. The filer checks the token under
+// the path lock the delete runs under, so a file that took the path since —
+// the table was dropped and another create staged the same name — is refused
+// with errStagedFileReplaced instead of deleted. A path that holds nothing is
+// not an error.
+func (s *Server) deleteStagedMetadataFile(ctx context.Context, bucketName, tablePath, metadataFileName, token string) error {
+	if token == "" {
+		return fmt.Errorf("%w: %s has no stage token", errStagedFileReplaced, metadataFileName)
+	}
+	opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	metadataDir := metadataDirPath(bucketName, tablePath)
+	return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		_, err := filer_pb.DoRemoveRequest(opCtx, client, &filer_pb.DeleteEntryRequest{
+			Directory:            metadataDir,
+			Name:                 metadataFileName,
+			IsDeleteData:         true,
+			IgnoreRecursiveError: true,
+			ExpectedExtended:     map[string][]byte{stageTokenKey: []byte(token)},
+		})
+		if status.Code(err) == codes.FailedPrecondition {
+			return fmt.Errorf("%w: %s/%s: %v", errStagedFileReplaced, metadataDir, metadataFileName, err)
+		}
+		return err
+	})
+}
+
 func (s *Server) loadMetadataFile(ctx context.Context, bucketName, tablePath, metadataFileName string) ([]byte, error) {
 	opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()

weed/s3api/s3tables/filer_ops.go

@@ -17,6 +17,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -26,6 +27,11 @@ var (
 	ErrConcurrentUpdate  = errors.New("attribute changed concurrently")
 	ErrNameTaken         = errors.New("catalog name is taken")
 	ErrLocationInUse     = errors.New("location is in use by another table or view")
+	// ErrPublishedRecordChanged is the ErrConcurrentUpdate a create returns
+	// when the record it wrote was already visible, and another caller changed
+	// it — updated it, or renamed it away — before the create could back it
+	// out. Whatever the create staged for that record may still be in use.
+	ErrPublishedRecordChanged = fmt.Errorf("published record changed before the create could back it out: %w", ErrConcurrentUpdate)
 )
 
 // extendedAttributeUpdateAttempts bounds the read-modify-write retry loop in
@@ -799,11 +805,18 @@ func (h *S3TablesHandler) confirmParentOnce(ctx context.Context, client filer_pb
 
 func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, recheck bool, childPath string, written map[string][]byte, emptyDirs []string) error {
 	parent, err := h.openParent(ctx, client, parentPath, isParent)
-	confirmed := false
+	// mayBeVisible is whether the mark may have come off: it did, or a write
+	// taking it off was sent and failed without the filer saying it refused
+	// it. A failure before any such write — the lookup that precedes it, say —
+	// leaves the mark on, and the record as invisible as it was.
+	confirmed, mayBeVisible := false, false
 	if err == nil {
 		// The mark comes off only while the record still holds what this
 		// create wrote; a failed write backs the record out like a closed parent.
-		if err = h.removeExtendedAttributesIf(ctx, client, childPath, written, ExtendedKeyDeleting); err == nil {
+		writes := &markRemovalWrites{SeaweedFilerClient: client}
+		err = h.removeExtendedAttributesIf(ctx, writes, childPath, written, ExtendedKeyDeleting)
+		mayBeVisible = err == nil || writes.uncertain
+		if err == nil {
 			confirmed = true
 			if recheck {
 				err = h.parentStillOpen(ctx, client, parentPath, isParent, parent)
@@ -813,7 +826,7 @@ func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.Seaw
 	if err == nil {
 		return nil
 	}
-	err = h.backOutChild(ctx, client, childPath, written, confirmed, emptyDirs, err)
+	err = h.backOutChild(ctx, client, childPath, written, confirmed, mayBeVisible, emptyDirs, err)
 	if errors.Is(err, errParentGone) {
 		h.removeLeftoverDirectory(context.WithoutCancel(ctx), client, parentPath)
 	}
@@ -821,8 +834,11 @@ func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.Seaw
 }
 
 // backOutChild removes what a create wrote onto childPath, after the create
-// failed with err, and returns err, or why the record stays.
-func (h *S3TablesHandler) backOutChild(ctx context.Context, client filer_pb.SeaweedFilerClient, childPath string, written map[string][]byte, confirmed bool, emptyDirs []string, err error) error {
+// failed with err, and returns err, or why the record stays. mayBeVisible is
+// whether readers may have seen the record: a record that moved on is then
+// reported as ErrPublishedRecordChanged, since what was staged for it may
+// still be in use.
+func (h *S3TablesHandler) backOutChild(ctx context.Context, client filer_pb.SeaweedFilerClient, childPath string, written map[string][]byte, confirmed, mayBeVisible bool, emptyDirs []string, err error) error {
 	ctx = context.WithoutCancel(ctx)
 	expected := written
 	keys := make([]string, 0, len(written))
@@ -845,7 +861,12 @@ func (h *S3TablesHandler) backOutChild(ctx context.Context, client filer_pb.Seaw
 		glog.Warningf("back out %s: %v", childPath, backOutErr)
 		if errors.Is(backOutErr, ErrConcurrentUpdate) {
 			// The record moved past this create — adopted or rewritten — so
-			// there is nothing of its own left to remove.
+			// there is nothing of its own left to remove. While it was still
+			// marked, only another create could take it over; once readers
+			// could see it, an update or a rename may have built on it.
+			if mayBeVisible {
+				return fmt.Errorf("%w: %s", ErrPublishedRecordChanged, childPath)
+			}
 			return fmt.Errorf("%w: %s changed during create", ErrConcurrentUpdate, childPath)
 		}
 		return fmt.Errorf("%w: %s: %v (%v)", errBackOutFailed, childPath, backOutErr, err)
@@ -856,6 +877,24 @@ func (h *S3TablesHandler) backOutChild(ctx context.Context, client filer_pb.Seaw
 	return err
 }
 
+// markRemovalWrites passes a create's mark-removal calls through to the filer
+// and records whether an UpdateEntry was sent whose outcome is not a refusal:
+// it succeeded, or it failed in a way that does not say whether it landed. A
+// compare the filer refused (FailedPrecondition) or an entry it no longer has
+// (NotFound) changed nothing.
+type markRemovalWrites struct {
+	filer_pb.SeaweedFilerClient
+	uncertain bool
+}
+
+func (w *markRemovalWrites) UpdateEntry(ctx context.Context, in *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
+	resp, err := w.SeaweedFilerClient.UpdateEntry(ctx, in, opts...)
+	if code := status.Code(err); code != codes.FailedPrecondition && code != codes.NotFound {
+		w.uncertain = true
+	}
+	return resp, err
+}
+
 // dirListPageSize is how many entries dirHoldsFiles reads per ListEntries call.
 const dirListPageSize = 1024
 
@@ -1055,7 +1094,8 @@ func (h *S3TablesHandler) claimCatalogEntryAt(ctx context.Context, client filer_
 		return err
 	}
 	if err := checkLocationUnclaimed(ctx, client, bucketName, path, metadataLocation); err != nil {
-		return h.backOutChild(ctx, client, path, attrs, false, emptyDirs, err)
+		// The claim still carries its mark, so no reader has seen the record.
+		return h.backOutChild(ctx, client, path, attrs, false, false, emptyDirs, err)
 	}
 	return nil
 }

weed/s3api/s3tables/handler_table.go

@@ -178,6 +178,10 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
 			h.writeError(w, http.StatusConflict, ErrCodeTableAlreadyExists, fmt.Sprintf("table %s already exists", tableName))
 			return err
 		}
+		if errors.Is(err, ErrPublishedRecordChanged) {
+			h.writeError(w, http.StatusConflict, ErrCodeConflict, PublishedTableChangedMessage)
+			return err
+		}
 		if errors.Is(err, ErrConcurrentUpdate) {
 			h.writeError(w, http.StatusConflict, ErrCodeConflict, "table changed during create, retry the request")
 			return err

weed/s3api/s3tables/table_data_dir_test.go

@@ -24,6 +24,7 @@ func TestTableDataDirFromMetadataLocation(t *testing.T) {
 
 func TestMetadataVersionFromLocationUniqueSuffix(t *testing.T) {
 	cases := map[string]int{
+		"s3://bkt/ns/t/metadata/v1-0a1b2c3d-4e5f-6789-abcd-ef0123456789.metadata.json": 1,
 		"s3://bkt/ns/t/metadata/v4.metadata.json":                                      4,
 		"s3://bkt/ns/t/metadata/v4-0a1b2c3d-4e5f-6789-abcd-ef0123456789.metadata.json": 4,
 		"s3://bkt/ns/t/metadata/00007-0a1b2c3d.metadata.json":                          7,

weed/s3api/s3tables/types.go

@@ -2,6 +2,7 @@ package s3tables
 
 import (
 	"encoding/json"
+	"errors"
 	"strings"
 	"time"
 )
@@ -589,6 +590,23 @@ func (e *S3TablesError) Error() string {
 	return e.Message
 }
 
+// PublishedTableChangedMessage is the message of the conflict a CreateTable
+// answers when the table it registered was already visible, and another
+// caller updated or renamed it before the create could back it out. The
+// record survives, under this name or another, so whatever the caller staged
+// for it may still be in use.
+const PublishedTableChangedMessage = "table was registered and then changed by another request before the create finished"
+
+// IsPublishedTableChanged reports whether err, as a CreateTable caller
+// receives it, is the conflict PublishedTableChangedMessage describes.
+func IsPublishedTableChanged(err error) bool {
+	if errors.Is(err, ErrPublishedRecordChanged) {
+		return true
+	}
+	var tableErr *S3TablesError
+	return errors.As(err, &tableErr) && tableErr.Type == ErrCodeConflict && tableErr.Message == PublishedTableChangedMessage
+}
+
 // Table formats a catalog entry may declare.
 //
 // ICEBERG tables carry metadata the catalog maintains and the maintenance