filer.remote.gateway: keep the remote object while a version under a delete marker is locked (#12459)

* filer.remote.gateway: keep the remote object while a version under a delete marker is locked

Every version of a versioned bucket uploads to the object's key, so on a
remote without versioning that object is the only remote copy of the
latest version. A delete marker deleted it even when a version still held
a legal hold or unexpired retention. Keep it while any version of the key
is locked, the same rule the filer uses before pruning a remote entry.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.remote.gateway, filer.remote.sync: say that earlier versions need remote versioning

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.remote.gateway: delete a kept delete marker's object once the version lock is released

A delete marker kept for a locked version stays unstamped. When an
update or delete leaves a version that carried a legal hold or retention
without an active lock, look up the key's latest version and, if it is
still that unstamped marker, run the delete again. Only changes to
versions that carried a lock trigger it, so the noncurrent mark and tag
updates of unlocked versions do not send repeat remote deletes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.remote.gateway: test the kept delete marker retry through the bucket mount

The retry reads the key's versions through a filer client parameter, as
the rename helpers do, so a test can check the mapped object key and the
marker stamp, and that lock-releasing updates and deletes reach it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.remote.gateway: check the lock release lookup goes to the gateway's filer

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

weed/command/filer_remote_sync_dir.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_objectlock"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -367,6 +368,10 @@ func processDeleteEvent(
 	// Skip deletion of internal version files; individual version
 	// deletes should not propagate to the remote object
 	if isVersionedPath(resp.Directory, message.OldEntry.Name, message.OldEntry.IsDirectory) {
+		if newParent, newName, ok := rewriteVersionedSourcePath(resp.Directory, message.OldEntry.Name); ok && releasesVersionLock(message.OldEntry, nil) {
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation)
+			return syncKeptDeleteMarker(client, filerSource, resp.Directory, dest)
+		}
 		glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
 		return nil
 	}
@@ -481,6 +486,10 @@ func processUpdateEvent(
 		return nil
 	}
 	if isVersionedPath(message.NewParentPath, message.NewEntry.Name, message.NewEntry.IsDirectory) {
+		if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok && releasesVersionLock(message.OldEntry, message.NewEntry) {
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation)
+			return syncKeptDeleteMarker(client, filerClient, message.NewParentPath, dest)
+		}
 		glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
 		return nil
 	}
@@ -1369,12 +1378,30 @@ func isDeleteMarker(entry *filer_pb.Entry) bool {
 
 // syncDeleteMarker propagates a delete marker to the remote storage and
 // persists a local sync marker so that replaying the same event is a no-op.
+// Every version uploads to the object's key, so while a version under the
+// marker holds a legal hold or retention the remote object can be its only
+// remote copy: it is kept, and the marker stays unstamped for
+// syncKeptDeleteMarker to retry.
 func syncDeleteMarker(
 	client remote_storage.RemoteStorageClient,
 	filerClient filer_pb.FilerClient,
 	message *filer_pb.EventNotification,
 	dest *remote_pb.RemoteStorageLocation,
 ) error {
+	var locked bool
+	if err := filerClient.WithFilerClient(true, func(c filer_pb.SeaweedFilerClient) error {
+		ctx, cancel := lookupContext()
+		defer cancel()
+		var err error
+		locked, err = s3_objectlock.HasObjectsWithActiveLocks(ctx, c, message.NewParentPath)
+		return err
+	}); err != nil {
+		return err
+	}
+	if locked {
+		glog.V(0).Infof("keep %s: a version under the delete marker is locked", remote_storage.FormatLocation(dest))
+		return nil
+	}
 	glog.V(0).Infof("delete (marker) %s", remote_storage.FormatLocation(dest))
 	if err := deleteRemoteFile(client, dest); err != nil {
 		return err
@@ -1385,6 +1412,43 @@ func syncDeleteMarker(
 	})
 }
 
+// syncKeptDeleteMarker retries the delete of the latest delete marker in
+// versionsDir that syncDeleteMarker kept for a locked version.
+func syncKeptDeleteMarker(
+	client remote_storage.RemoteStorageClient,
+	filerClient filer_pb.FilerClient,
+	versionsDir string,
+	dest *remote_pb.RemoteStorageLocation,
+) error {
+	dir, name := util.FullPath(versionsDir).DirAndName()
+	versionsEntry, err := currentEntry(filerClient, dir, name)
+	if err != nil || versionsEntry == nil {
+		return err
+	}
+	markerName := string(versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey])
+	if string(versionsEntry.Extended[s3_constants.ExtLatestVersionIsDeleteMarker]) != "true" || markerName == "" {
+		return nil
+	}
+	marker, err := currentEntry(filerClient, versionsDir, markerName)
+	if err != nil || !isDeleteMarker(marker) || marker.RemoteEntry != nil {
+		return err
+	}
+	return syncDeleteMarker(client, filerClient, &filer_pb.EventNotification{NewParentPath: versionsDir, NewEntry: marker}, dest)
+}
+
+// releasesVersionLock reports an update or delete of a version that carried a
+// legal hold or retention and holds no active lock afterwards.
+func releasesVersionLock(oldEntry, newEntry *filer_pb.Entry) bool {
+	if oldEntry == nil || oldEntry.IsDirectory || isDeleteMarker(oldEntry) {
+		return false
+	}
+	hold := strings.TrimSpace(strings.ToUpper(string(oldEntry.Extended[s3_constants.ExtLegalHoldKey])))
+	if hold != s3_constants.LegalHoldOn && len(oldEntry.Extended[s3_constants.ExtObjectLockModeKey]) == 0 {
+		return false
+	}
+	return !s3_objectlock.EntryHasActiveLock(newEntry, time.Now())
+}
+
 // isVersionedPath returns true if the dir/name refers to an internal
 // versioning path (.versions directory or a version file inside it).
 // These paths are SeaweedFS-internal and must not be synced to remote