seaweedfs/weed/s3api/s3api_object_handlers_delete.go

package s3api

import (
	"context"
	"encoding/xml"
	"errors"
	"io"
	"net/http"
	"strings"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

const (
	deleteMultipleObjectsLimit = 1000
)

type deleteMutationResult struct {
	versionId    string
	deleteMarker bool
}

func deleteErrorFromCode(code s3err.ErrorCode, key, versionId string) DeleteError {
	apiErr := s3err.GetAPIError(code)
	return DeleteError{
		Code:      apiErr.Code,
		Message:   apiErr.Description,
		Key:       key,
		VersionId: versionId,
	}
}

func validateDeleteObjectIdentifier(object ObjectIdentifier) s3err.ErrorCode {
	if !s3_constants.IsValidObjectKey(object.Key) || !isValidVersionID(object.VersionId) {
		return s3err.ErrInvalidRequest
	}
	return s3err.ErrNone
}

// isMissingDeleteConditionTarget normalizes missing-target detection for conditional deletes.
// Prefer errors.Is(err, filer_pb.ErrNotFound) and errors.Is(err, ErrDeleteMarker); keep the
// string-based fallback only as a defensive bridge for filer paths that still return plain text.
func isMissingDeleteConditionTarget(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrDeleteMarker) {
		return true
	}

	lowerErr := strings.ToLower(err.Error())
	return strings.Contains(lowerErr, "not found")
}

func (s3a *S3ApiServer) resolveDeleteConditionalEntry(bucket, object, versionId, versioningState string) (*filer_pb.Entry, error) {
	if !isValidVersionID(versionId) {
		return nil, errInvalidVersionID
	}
	normalizedObject := s3_constants.NormalizeObjectKey(object)

	if versionId != "" {
		if versionId == "null" {
			return s3a.getObjectEntryRoutedByKey(bucket, normalizedObject)
		}
		return s3a.getObjectScopedEntryRoutedByKey(bucket, normalizedObject, s3a.getVersionedObjectDir(bucket, normalizedObject), s3a.getVersionFileName(versionId))
	}

	switch versioningState {
	case s3_constants.VersioningEnabled:
		entry, err := s3a.getLatestObjectVersion(bucket, normalizedObject)
		if err != nil {
			return nil, err
		}
		return normalizeConditionalTargetEntry(entry), nil
	default:
		entry, err := s3a.resolveObjectEntry(bucket, normalizedObject, "")
		if err != nil {
			return nil, err
		}
		return normalizeConditionalTargetEntry(entry), nil
	}
}

func (s3a *S3ApiServer) validateDeleteIfMatch(entry *filer_pb.Entry, ifMatch string, missingCode s3err.ErrorCode) s3err.ErrorCode {
	if ifMatch == "" {
		return s3err.ErrNone
	}
	if entry == nil {
		return missingCode
	}
	if ifMatch == "*" {
		return s3err.ErrNone
	}
	if !s3a.etagMatches(ifMatch, s3a.getObjectETag(entry)) {
		return s3err.ErrPreconditionFailed
	}
	return s3err.ErrNone
}

func (s3a *S3ApiServer) checkDeleteIfMatch(bucket, object, versionId, versioningState, ifMatch string, missingCode s3err.ErrorCode) s3err.ErrorCode {
	if ifMatch == "" {
		return s3err.ErrNone
	}

	entry, err := s3a.resolveDeleteConditionalEntry(bucket, object, versionId, versioningState)
	if err != nil {
		if isMissingDeleteConditionTarget(err) {
			return missingCode
		}
		glog.Errorf("checkDeleteIfMatch: failed to resolve %s/%s (versionId=%s): %v", bucket, object, versionId, err)
		return s3err.ErrInternalError
	}

	return s3a.validateDeleteIfMatch(entry, ifMatch, missingCode)
}

func (s3a *S3ApiServer) deleteVersionedObject(r *http.Request, bucket, object, versionId, versioningState string) (deleteMutationResult, s3err.ErrorCode) {
	var result deleteMutationResult

	// The key "dir/" is the filer directory itself, which a delete marker cannot stand
	// in for without hiding the children underneath it. It is not a versioned object,
	// so it is deleted the way an unversioned bucket deletes it.
	if versionId == "" && strings.HasSuffix(object, "/") {
		return result, s3a.deleteDirectoryMarker(r, bucket, object)
	}

	switch {
	case versionId != "":
		versionEntry, versionLookupErr := s3a.getSpecificObjectVersion(bucket, object, versionId)
		if versionLookupErr == nil && versionEntry != nil && versionEntry.Extended != nil {
			if deleteMarker, ok := versionEntry.Extended[s3_constants.ExtDeleteMarkerKey]; ok && string(deleteMarker) == "true" {
				result.deleteMarker = true
			}
		}
		governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
		if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
			glog.V(2).Infof("deleteVersionedObject: object lock check failed for %s/%s version %s: %v", bucket, object, versionId, err)
			return result, s3err.ErrAccessDenied
		}
		if err := s3a.deleteSpecificObjectVersion(r.Context(), bucket, object, versionId, false); err != nil {
			glog.Errorf("deleteVersionedObject: failed to delete specific version %s for %s/%s: %v", versionId, bucket, object, err)
			return result, s3err.ErrInternalError
		}
		result.versionId = versionId
		return result, s3err.ErrNone

	case versioningState == s3_constants.VersioningEnabled:
		deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
		if err != nil {
			glog.Errorf("deleteVersionedObject: failed to create delete marker for %s/%s: %v", bucket, object, err)
			return result, s3err.ErrInternalError
		}
		result.versionId = deleteMarkerVersionId
		result.deleteMarker = true
		return result, s3err.ErrNone

	case versioningState == s3_constants.VersioningSuspended:
		governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
		if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
			glog.V(2).Infof("deleteVersionedObject: object lock check failed for %s/%s null version: %v", bucket, object, err)
			return result, s3err.ErrAccessDenied
		}
		// Delete the null version on the owner the routed reads consult: on
		// the default filer an absent entry is a successful delete, so a null
		// version that only reached the owner would survive a reported
		// success. An unsettled routed attempt falls back to the lock path.
		nullDeleted := false
		if owner := s3a.routableWriteOwner(bucket, object); owner != "" {
			worm, lockErr := s3a.isObjectLockEnabled(bucket)
			if lockErr == nil {
				code, settled := s3a.routedDeleteNullVersion(owner, bucket, object, worm, worm && governanceBypassAllowed, false)
				if settled {
					if code != s3err.ErrNone {
						return result, code
					}
					nullDeleted = true
				}
			}
		}
		if !nullDeleted {
			if err := s3a.deleteSpecificObjectVersion(r.Context(), bucket, object, "null", false); err != nil {
				glog.Errorf("deleteVersionedObject: failed to delete null version for %s/%s: %v", bucket, object, err)
				return result, s3err.ErrInternalError
			}
		}
		// Suspended versioning overwrites the null version with a single null delete
		// marker (S3 spec), so the marker replaces any prior one instead of piling up.
		if err := s3a.createNullDeleteMarker(bucket, object); err != nil {
			glog.Errorf("deleteVersionedObject: failed to create null delete marker for suspended versioning %s/%s: %v", bucket, object, err)
			return result, s3err.ErrInternalError
		}
		result.versionId = "null"
		result.deleteMarker = true
		return result, s3err.ErrNone
	}

	glog.Errorf("deleteVersionedObject: unsupported versioning state %q for %s/%s", versioningState, bucket, object)
	return result, s3err.ErrInternalError
}

// deleteUnversionedObjectWithClient removes the bare object entry. When
// metadataOnly is true the filer skips per-chunk DeleteFile RPCs and
// relies on the volume's natural TTL to reclaim chunks; pass true only
// when the entry's Attributes.TtlSec > 0 so the volume is guaranteed to
// drop the chunks on its own.
func (s3a *S3ApiServer) deleteUnversionedObjectWithClient(ctx context.Context, client filer_pb.SeaweedFilerClient, bucket, object string, metadataOnly bool) error {
	if !s3_constants.IsValidBucketName(bucket) || !s3_constants.IsValidObjectKey(object) {
		return errors.New("invalid bucket or object path")
	}
	target := util.NewFullPath(s3a.bucketDir(bucket), object)
	dir, name := target.DirAndName()
	// The caller holds one client for a whole batch, so a dropped reply is
	// replayed on that client rather than by re-entering WithFilerClient.
	return retryFilerOp(ctx, "delete "+string(target), func() error {
		return deleteObjectEntry(ctx, client, dir, name, !metadataOnly, false)
	})
}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := s3_constants.GetBucketAndObject(r)
	glog.Infof("DeleteObjectHandler %s %s", bucket, object)
	// The filer ops below each retry, and a failover walk runs the whole set
	// once per filer, so the backoff comes out of one allowance held here.
	r = r.WithContext(withFilerRetryBudget(r.Context(), filerRetryRequestBudget))
	if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, object); err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
		return
	}

	// Check for specific version ID in query parameters
	versionId := r.URL.Query().Get("versionId")

	// Get detailed versioning state for proper handling of suspended vs enabled versioning
	versioningState, err := s3a.getVersioningState(bucket)
	if err != nil {
		if err == filer_pb.ErrNotFound {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	versioningConfigured := (versioningState != "")

	if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	if errCode := s3a.recheckTagConditionedPolicyWithLookup(r, bucket, object, versionId, string(s3_constants.ACTION_WRITE), "DeleteObjectHandler"); errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	var auditLog *s3err.AccessLog
	if s3err.Logger != nil {
		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
	}

	if ifMatchResult := s3a.checkDeleteIfMatch(bucket, object, versionId, versioningState, r.Header.Get(s3_constants.IfMatch), s3err.ErrPreconditionFailed); ifMatchResult != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, ifMatchResult)
		return
	}

	var deleteResult deleteMutationResult
	var deleteCode s3err.ErrorCode

	// A trailing-slash key is a directory marker in every bucket, versioned or not, and
	// is deleted the same way: the raw delete below cannot handle a directory that still
	// has children, and versioning has nothing to add to a key that is not an object.
	deleteHandled := false
	if versionId == "" && strings.HasSuffix(object, "/") {
		deleteCode, deleteHandled = s3a.withObjectWriteLock(bucket, object, func() s3err.ErrorCode {
			return s3a.checkDeleteIfMatch(bucket, object, versionId, versioningState, r.Header.Get(s3_constants.IfMatch), s3err.ErrPreconditionFailed)
		}, func() s3err.ErrorCode {
			return s3a.deleteDirectoryMarker(r, bucket, object)
		}), true
	}

	// Fast path: route the delete to the owner filer under its per-path lock;
	// routedObjectOwner excludes versioned/object-lock buckets.
	if !deleteHandled && !versioningConfigured {
		if cond, condOk := buildDeleteCondition(r); condOk {
			if owner, ownerOk := s3a.routedObjectOwner(bucket, object); ownerOk {
				resp, err := s3a.routedDelete(owner, bucket, object, cond)
				switch {
				case err != nil:
					glog.Warningf("DeleteObjectHandler: routed delete to %s failed for %s/%s, falling back to lock: %v", owner, bucket, object, err)
				case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
					deleteCode, deleteHandled = s3err.ErrPreconditionFailed, true
				case resp.Error != "":
					// Non-precondition error: fall back (the lock path handles cases
					// the raw delete cannot, e.g. a non-empty directory marker).
					glog.Warningf("DeleteObjectHandler: routed delete to %s returned %q for %s/%s, falling back to lock", owner, resp.Error, bucket, object)
				default:
					deleteCode, deleteHandled = s3err.ErrNone, true
				}
			}
		}
	}
	// Versioned/suspended delete with no specific version: route off the lock when
	// the bucket has an owner. createDeleteMarker routes its own pointer flip; a
	// delete marker never removes a locked version, so object-lock buckets route
	// here too. The If-Match precondition was already checked above.
	if !deleteHandled && versioningConfigured && versionId == "" {
		if owner := s3a.routableWriteOwner(bucket, object); owner != "" {
			deleteResult, deleteCode = s3a.deleteVersionedObject(r, bucket, object, versionId, versioningState)
			deleteHandled = true
		}
	}
	// Specific-version delete: route off the lock. A real version recomputes the
	// .versions pointer excluding it and deletes the version file; the null
	// version is the regular object entry, deleted directly. Object-lock buckets
	// gate the delete on the version's WORM guards, evaluated on the owner — for
	// governance bypass the retention guard is scoped to COMPLIANCE so the filer
	// allows a governance-mode delete while still denying compliance and legal
	// hold, without the gateway reading the version.
	if !deleteHandled && versionId != "" {
		worm, lockErr := s3a.isObjectLockEnabled(bucket)
		bypass := worm && s3a.evaluateGovernanceBypassRequest(r, bucket, object)
		if lockErr == nil {
			if owner := s3a.objectWriteOwner(bucket, object); owner != "" {
				deleteResult.versionId = versionId
				if ve, vErr := s3a.getSpecificObjectVersion(bucket, object, versionId); vErr == nil && ve != nil && ve.Extended != nil {
					if dm, ok := ve.Extended[s3_constants.ExtDeleteMarkerKey]; ok && string(dm) == "true" {
						deleteResult.deleteMarker = true
					}
				}
				if versionId == "null" {
					deleteCode, deleteHandled = s3a.routedDeleteNullVersion(owner, bucket, object, worm, bypass, false)
				} else {
					deleteCode, deleteHandled = s3a.routedDeleteSpecificVersion(owner, bucket, object, versionId, worm, bypass, false), true
				}
			}
		}
	}
	if !deleteHandled {
		deleteCode = s3a.withObjectWriteLock(bucket, object, func() s3err.ErrorCode {
			return s3a.checkDeleteIfMatch(bucket, object, versionId, versioningState, r.Header.Get(s3_constants.IfMatch), s3err.ErrPreconditionFailed)
		}, func() s3err.ErrorCode {
			if versioningConfigured {
				result, errCode := s3a.deleteVersionedObject(r, bucket, object, versionId, versioningState)
				if errCode != s3err.ErrNone {
					return errCode
				}
				deleteResult = result
				return s3err.ErrNone
			}

			governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
			if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
				glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
				return s3err.ErrAccessDenied
			}

			if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
				return s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, object, false)
			}); err != nil {
				glog.Errorf("DeleteObjectHandler: failed to delete %s/%s: %v", bucket, object, err)
				return s3err.ErrInternalError
			}

			return s3err.ErrNone
		})
	}
	if deleteCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, deleteCode)
		return
	}

	if deleteResult.versionId != "" {
		w.Header().Set("x-amz-version-id", deleteResult.versionId)
	}
	if deleteResult.deleteMarker {
		w.Header().Set("x-amz-delete-marker", "true")
	}

	if auditLog != nil {
		auditLog.Key = strings.TrimPrefix(object, "/")
		s3err.PostAccessLog(*auditLog)
		s3err.MarkAuditLogged(r)
	}

	stats_collect.RecordBucketActiveTime(bucket)
	stats_collect.S3DeletedObjectsCounter.WithLabelValues(bucket).Inc()
	w.WriteHeader(http.StatusNoContent)
}

// ObjectIdentifier represents an object to be deleted with its key name and optional version ID.
type ObjectIdentifier struct {
	Key                   string `xml:"Key"`
	VersionId             string `xml:"VersionId,omitempty"`
	ETag                  string `xml:"ETag,omitempty"`
	DeleteMarker          bool   `xml:"DeleteMarker,omitempty"`
	DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId,omitempty"`
}

// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
	// Element to enable quiet mode for the request
	Quiet bool
	// List of objects to be deleted
	Objects []ObjectIdentifier `xml:"Object"`
}

// DeleteError structure.
type DeleteError struct {
	Code      string `xml:"Code"`
	Message   string `xml:"Message"`
	Key       string `xml:"Key"`
	VersionId string `xml:"VersionId,omitempty"`
}

// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`

	// Collection of all deleted objects
	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`

	// Collection of errors deleting certain objects.
	Errors []DeleteError `xml:"Error,omitempty"`
}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := s3_constants.GetBucketAndObject(r)
	glog.V(3).Infof("DeleteMultipleObjectsHandler %s", bucket)

	deleteXMLBytes, err := io.ReadAll(r.Body)
	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	deleteObjects := &DeleteObjectsRequest{}
	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
		return
	}

	if len(deleteObjects.Objects) > deleteMultipleObjectsLimit {
		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
		return
	}

	var deletedObjects []ObjectIdentifier
	var deleteErrors []DeleteError
	var auditLog *s3err.AccessLog

	if s3err.Logger != nil {
		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
	}

	// Get detailed versioning state for proper handling of suspended vs enabled versioning
	versioningState, err := s3a.getVersioningState(bucket)
	if err != nil {
		if err == filer_pb.ErrNotFound {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	versioningConfigured := (versioningState != "")
	deletedCount := 0

	// Per-key authorization: keys arrive in the body, so the route Auth middleware
	// only authenticated. Authorize each key via AuthorizeObjectDelete below.
	var identity *Identity
	if id := s3_constants.GetIdentityFromContext(r); id != nil {
		identity, _ = id.(*Identity)
	}

	// The keys below each drive their own bounded filer retries, and the client
	// picks how many keys there are, so the whole batch shares one allowance.
	r = r.WithContext(withFilerRetryBudget(r.Context(), filerRetryRequestBudget))

	err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		// delete file entries
		for _, object := range deleteObjects.Objects {
			if object.Key == "" {
				continue
			}
			if validationCode := validateDeleteObjectIdentifier(object); validationCode != s3err.ErrNone {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(validationCode, object.Key, object.VersionId))
				continue
			}
			key := s3_constants.NormalizeObjectKey(object.Key)
			if key == "" {
				// A key of only slashes can never name a stored entry; like a
				// missing key the delete is an idempotent no-op, and a
				// conditional delete has no target to match.
				if object.ETag != "" {
					deleteErrors = append(deleteErrors, deleteErrorFromCode(s3err.ErrNoSuchKey, object.Key, object.VersionId))
					continue
				}
				deletedCount++
				if !deleteObjects.Quiet {
					deletedObjects = append(deletedObjects, ObjectIdentifier{Key: object.Key, VersionId: object.VersionId})
				}
				continue
			}
			if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, key); err != nil {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(s3err.ErrAccessDenied, object.Key, object.VersionId))
				continue
			}
			if authErr := s3a.iam.AuthorizeObjectDelete(r, identity, bucket, key, object.VersionId); authErr != s3err.ErrNone {
				// Defer to the recheck, mirroring the copy-source fallback: a
				// tag-conditioned Allow may still grant what phase one denied.
				if authErr != s3err.ErrAccessDenied || s3a.recheckPolicyForKeyWithLookup(r, http.MethodDelete, bucket, key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler") != s3err.ErrNone {
					deleteErrors = append(deleteErrors, deleteErrorFromCode(authErr, object.Key, object.VersionId))
					continue
				}
			}
			if errCode := s3a.recheckTagConditionedPolicyWithLookup(objectKeyPolicyRequest(r, http.MethodDelete, bucket, key, object.VersionId), bucket, key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler"); errCode != s3err.ErrNone {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(errCode, object.Key, object.VersionId))
				continue
			}

			var deleteResult deleteMutationResult
			deleteCode := s3a.withObjectWriteLock(bucket, key, func() s3err.ErrorCode {
				return s3a.checkDeleteIfMatch(bucket, key, object.VersionId, versioningState, object.ETag, s3err.ErrNoSuchKey)
			}, func() s3err.ErrorCode {
				if versioningConfigured {
					result, errCode := s3a.deleteVersionedObject(r, bucket, key, object.VersionId, versioningState)
					if errCode != s3err.ErrNone {
						return errCode
					}
					deleteResult = result
					return s3err.ErrNone
				}

				governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, key)
				if err := s3a.enforceObjectLockProtections(r, bucket, key, "", governanceBypassAllowed); err != nil {
					glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s: %v", bucket, key, err)
					return s3err.ErrAccessDenied
				}

				if strings.HasSuffix(key, "/") {
					return s3a.deleteDirectoryMarker(r, bucket, key)
				}

				if err := s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, key, false); err != nil {
					glog.Errorf("DeleteMultipleObjectsHandler: failed to delete %s/%s: %v", bucket, key, err)
					return s3err.ErrInternalError
				}

				return s3err.ErrNone
			})
			if deleteCode != s3err.ErrNone {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(deleteCode, object.Key, object.VersionId))
				continue
			}

			deletedCount++
			if !deleteObjects.Quiet {
				deletedObject := ObjectIdentifier{
					Key:       object.Key,
					VersionId: deleteResult.versionId,
				}
				if deleteResult.deleteMarker {
					deletedObject.DeleteMarker = true
					deletedObject.DeleteMarkerVersionId = deleteResult.versionId
					deletedObject.VersionId = ""
				}
				deletedObjects = append(deletedObjects, deletedObject)
			}

			if auditLog != nil {
				auditLog.Key = object.Key
				s3err.PostAccessLog(*auditLog)
				s3err.MarkAuditLogged(r)
			}
		}

		// Note: Empty folder cleanup is now handled asynchronously by EmptyFolderCleaner
		// which listens to metadata events and uses consistent hashing for coordination

		return nil
	})
	if err != nil {
		glog.Errorf("DeleteMultipleObjectsHandler: failed to initialize filer client for bucket %s: %v", bucket, err)
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	deleteResp := DeleteObjectsResponse{}
	if !deleteObjects.Quiet {
		deleteResp.DeletedObjects = deletedObjects
	}
	deleteResp.Errors = deleteErrors
	stats_collect.RecordBucketActiveTime(bucket)
	stats_collect.S3DeletedObjectsCounter.WithLabelValues(bucket).Add(float64(deletedCount))

	writeSuccessResponseXML(w, r, deleteResp)

}