filer.remote.sync: dispatch an update event by the side of the mount it touches instead of slicing the mount prefix off a path outside it (#11958)

* filer.remote.sync: dispatch an update event by the side of the mount it touches instead of slicing the mount prefix off a path outside it The daemon subscribes to its mount prefix and to /etc/remote. An update event whose path lies outside the mount still reaches processUpdateEvent: an update of the /etc/remote directory entry itself (which fs.meta.load of a mapping dump re-creates) through the /etc/remote prefix, or a rename across the mount boundary through the path on the inside. processUpdateEvent then maps both paths to remote keys by slicing the mount prefix off, which panics with "slice bounds out of range [20:11]" on a path shorter than the mount and names a key in the wrong tree on a longer one. The event sits in the replay window, so every restart dies on it and the persisted offset never moves. Before dispatch, an update event is now mapped onto the side the mount covers: a rename into the mount is the creation of the new entry, a rename out of it the deletion of the old entry's object (kept when the entry is remote-only and the object holds its only copy), an event with neither path inside is not the mount's and is dropped, and an update with both paths inside is unchanged. * filer.remote.sync: extract processCreateEvent Moves the create branch of eachEntryFunc into a function, matching processDeleteEvent and processUpdateEvent, so the create path can be exercised directly in tests. No behavior change. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.remote.sync: upload a create reshaped from a rename into the mount A file synced under the mount keeps its RemoteEntry across renames. The rename out deletes the remote object, and the rename back in reshapes to a create whose inherited stamp still matches the entry, so shouldSendToRemote skipped the upload and the new key stayed empty. The reshaped create now sends unconditionally: the stamp describes the old key, not this one. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.remote.sync: a create reshaped from a rename into the mount writes an empty entry as a zero-byte object * filer.remote.sync: a create reshaped from a rename sends a stamped entry truncated since its sync A stamped entry still reports remote-only once truncated to zero (the stamp's RemoteSize stays positive), so the remote-only exemption skipped it and the new key got no zero-byte object. Gate the exemption on isDroppedLocalCopy: only an entry whose size still matches its stamp has its content already at the new key via the rename's remote copy. * filer.remote.sync: recover a superseded create reshaped from a rename-in A create reshaped from a rename into the mount whose write is superseded was skipped like a plain create, but no later event covers this destination: remote.uncache accepts the rename's inherited RemoteEntry stamp and drops the local chunks, and the update it logs skips the upload through isDroppedLocalCopy, while the source mount's sync still deletes the source object. The destination object is then missing and the content exists nowhere. recoverSupersededCreate finishes the event from the entry the filer holds now: a rewritten entry is left to its own event's upload or uploaded here on the stamp edge, and a remote-only entry is completed by copying the object its stamp describes -- resolved through a pending-rename marker under this mount, or through the filer's mount mappings to the source mount's remote location. --------- Co-authored-by: james <james@medable.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: chrislusf <chris.lu@gmail.com>
2026-10-24 23:10:41 +02:00 · 2026-10-23 22:43:39 -07:00
1 parent 5e01caa522
commit 57a122fbcf
@@ -140,63 +140,24 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 		if filer_pb.IsEmpty(resp) {
 			return nil
 		}
+		mustSend := false
+		var renameSrc util.FullPath
+		if message.OldEntry != nil && message.NewEntry != nil {
+			reshaped, inMount := reshapeCrossMountRename(mountedDir, resp)
+			if !inMount {
+				return nil
+			}
+			// a create reshaped from a rename into the mount can carry a
+			// RemoteEntry earned at the old path: it stamps the old key, so
+			// it cannot vouch for this destination
+			mustSend = filer_pb.IsCreate(reshaped)
+			if mustSend {
+				renameSrc = util.NewFullPath(resp.Directory, message.OldEntry.Name)
+			}
+			resp, message = reshaped, reshaped.EventNotification
+		}
 		if filer_pb.IsCreate(resp) {
-			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
-				return nil
-			}
-			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
-				glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
-				return nil
-			}
-			// Propagate delete markers as deletions on the remote.
-			// Delete markers are zero-content version entries, so they
-			// would be filtered out by the HasData check below.
-			if isDeleteMarker(message.NewEntry) {
-				if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok {
-					dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation)
-					return syncDeleteMarker(client, option, message, dest)
-				}
-				return nil
-			}
-			if !filer.HasData(message.NewEntry) {
-				return nil
-			}
-			glog.V(2).Infof("create: %+v", resp)
-			if !shouldSendToRemote(message.NewEntry) {
-				glog.V(2).Infof("skipping creating: %+v", resp)
-				return nil
-			}
-			// Rewrite internal versioning paths to the original S3 key
-			// to prevent double-versioning when central also has versioning enabled
-			parentPath, entryName := message.NewParentPath, message.NewEntry.Name
-			isRewrittenVersion := false
-			if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok {
-				glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName)
-				parentPath, entryName = newParent, newName
-				isRewrittenVersion = true
-			}
-			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation)
-			if message.NewEntry.IsDirectory {
-				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
-				return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, *option.storageClass))
-			}
-			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, *option.storageClass), dest)
-			if errors.Is(writeErr, errSuperseded) {
-				glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
-				return nil
-			}
-			if writeErr != nil {
-				return writeErr
-			}
-			// Skip updateLocalEntry for versioned rewrites: the logical
-			// object (e.g. file.xml) has no filer entry in versioned
-			// buckets, and stamping the internal v_* entry with a
-			// RemoteEntry for the logical key is semantically wrong.
-			// Replay is safe because S3 PutObject is idempotent.
-			if isRewrittenVersion {
-				return nil
-			}
-			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
+			return processCreateEvent(option, filerSource, *option.storageClass, client, mountedDir, remoteStorageMountLocation, resp, mustSend, renameSrc)
 		}
 		if filer_pb.IsDelete(resp) {
 			return processDeleteEvent(filerSource, client, mountedDir, remoteStorageMountLocation, resp)
@@ -210,6 +171,136 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 	return eachEntryFunc, nil
 }

+// reshapeCrossMountRename maps an update event onto the side of it the mount
+// covers. The subscription delivers a rename when either of its paths lies
+// under the mounted directory, and only a path under it maps to a remote
+// key: toRemoteStorageLocation slices the mount prefix off, which panics on
+// a shorter path and names a key in the wrong tree on a longer one. A rename
+// into the mount is the creation of the new entry, a rename out of it the
+// deletion of the old entry's object, unless that entry is remote-only and
+// the object holds its only copy. An update with both paths inside is
+// returned as is; one with neither is not the mount's (false).
+func reshapeCrossMountRename(mountedDir string, resp *filer_pb.SubscribeMetadataResponse) (*filer_pb.SubscribeMetadataResponse, bool) {
+	message := resp.EventNotification
+	oldInMount := isUnderMount(mountedDir, resp.Directory, message.OldEntry.Name)
+	newInMount := isUnderMount(mountedDir, message.NewParentPath, message.NewEntry.Name)
+	switch {
+	case oldInMount && newInMount:
+		return resp, true
+	case newInMount:
+		glog.V(0).Infof("rename into mount %s: %s/%s -> %s/%s, creating", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name)
+		return &filer_pb.SubscribeMetadataResponse{
+			Directory:         message.NewParentPath,
+			EventNotification: &filer_pb.EventNotification{NewEntry: message.NewEntry, NewParentPath: message.NewParentPath, Signatures: message.Signatures},
+			TsNs:              resp.TsNs,
+		}, true
+	case oldInMount:
+		if isRemoteOnly(message.OldEntry) {
+			glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, keeping the remote-only object", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name)
+			return nil, false
+		}
+		glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, deleting", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name)
+		return &filer_pb.SubscribeMetadataResponse{
+			Directory:         resp.Directory,
+			EventNotification: &filer_pb.EventNotification{OldEntry: message.OldEntry, Signatures: message.Signatures},
+			TsNs:              resp.TsNs,
+		}, true
+	default:
+		return nil, false
+	}
+}
+
+// isUnderMount reports whether dir/name is the mounted directory or lies
+// below it; a trailing slash on the mount flag does not change the answer.
+func isUnderMount(mountedDir string, dir string, name string) bool {
+	mount := strings.TrimSuffix(mountedDir, "/")
+	path := string(util.NewFullPath(dir, name))
+	return path == mount || strings.HasPrefix(path, mount+"/")
+}
+
+// processCreateEvent uploads a created entry to the remote key its path maps
+// to. mustSend skips the sync-stamp check: a create reshaped out of a rename
+// into the mount inherits a RemoteEntry stamped at the old path's key, which
+// says nothing about whether this destination holds the content. renameSrc is
+// the path that rename came from, and is empty for a genuine create.
+func processCreateEvent(
+	filerClient filer_pb.FilerClient,
+	filerSource filer_pb.FilerClient,
+	storageClass string,
+	client remote_storage.RemoteStorageClient,
+	mountedDir string,
+	remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
+	resp *filer_pb.SubscribeMetadataResponse,
+	mustSend bool,
+	renameSrc util.FullPath,
+) error {
+	message := resp.EventNotification
+	if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
+		return nil
+	}
+	if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+		glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
+		return nil
+	}
+	// Propagate delete markers as deletions on the remote.
+	// Delete markers are zero-content version entries, so they
+	// would be filtered out by the HasData check below.
+	if isDeleteMarker(message.NewEntry) {
+		if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok {
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation)
+			return syncDeleteMarker(client, filerClient, message, dest)
+		}
+		return nil
+	}
+	// A create reshaped from a rename into the mount (mustSend) writes an empty
+	// entry as a zero-byte object; a dropped local copy still skips since the
+	// rename already copied its object to the new key. A stamped entry whose
+	// size diverged since (a truncate) is not a dropped copy and sends its
+	// new content.
+	if !filer.HasData(message.NewEntry) && !(mustSend && !isDroppedLocalCopy(message.NewEntry)) {
+		return nil
+	}
+	glog.V(2).Infof("create: %+v", resp)
+	if !mustSend && !shouldSendToRemote(message.NewEntry) {
+		glog.V(2).Infof("skipping creating: %+v", resp)
+		return nil
+	}
+	// Rewrite internal versioning paths to the original S3 key
+	// to prevent double-versioning when central also has versioning enabled
+	parentPath, entryName := message.NewParentPath, message.NewEntry.Name
+	isRewrittenVersion := false
+	if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok {
+		glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName)
+		parentPath, entryName = newParent, newName
+		isRewrittenVersion = true
+	}
+	dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation)
+	if message.NewEntry.IsDirectory {
+		glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
+		return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, storageClass))
+	}
+	remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, storageClass), dest)
+	if errors.Is(writeErr, errSuperseded) {
+		glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
+		if mustSend {
+			return recoverSupersededCreate(filerClient, filerSource, client, mountedDir, remoteStorageMountLocation, message.NewParentPath, message.NewEntry.Name, renameSrc, dest, storageClass)
+		}
+		return nil
+	}
+	if writeErr != nil {
+		return writeErr
+	}
+	// Skip updateLocalEntry for versioned rewrites: the logical
+	// object (e.g. file.xml) has no filer entry in versioned
+	// buckets, and stamping the internal v_* entry with a
+	// RemoteEntry for the logical key is semantically wrong.
+	// Replay is safe because S3 PutObject is idempotent.
+	if isRewrittenVersion {
+		return nil
+	}
+	return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry)
+}
+
 // processDeleteEvent removes the remote object an entry mapped to. A remote
 // object that is already absent counts as deleted: the entry was never
 // uploaded (created and deleted faster than the sync ran, or a replay of an
@@ -651,6 +742,176 @@ func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.F
 	return updateLocalEntry(filerClient, dir, current, remoteEntry)
 }

+// recoverSupersededCreate finishes a create reshaped from a rename into the
+// mount when the filer has moved past the entry the event described. A plain
+// superseded create skips because the superseding event follows in the log; a
+// reshaped one has no later event for this destination. remote.uncache accepts
+// the rename's inherited stamp and drops the chunks, and the update it logs
+// skips the upload through isDroppedLocalCopy, so the content would exist
+// nowhere. What the filer holds now decides: local content uploads here only
+// when the superseding event will not (the stamp-edge case uploadCurrentEntry
+// covers), and remote-only content copies from the object its stamp
+// describes, the way completeRemoteOnlyRename finishes a rename.
+func recoverSupersededCreate(
+	filerClient filer_pb.FilerClient,
+	filerSource filer_pb.FilerClient,
+	client remote_storage.RemoteStorageClient,
+	mountedDir string,
+	remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
+	dir, name string,
+	renameSrc util.FullPath,
+	dest *remote_pb.RemoteStorageLocation,
+	storageClass string,
+) error {
+	current, err := currentEntry(filerSource, dir, name)
+	if err != nil {
+		return err
+	}
+	if current == nil || current.IsDirectory {
+		return nil
+	}
+	if isRemoteOnly(current) {
+		return copyRenameInSource(filerClient, client, mountedDir, remoteStorageMountLocation, dir, current, renameSrc, dest, storageClass)
+	}
+	if shouldSendToRemote(current) {
+		glog.V(0).Infof("leaving %s to the event that superseded the rename", remote_storage.FormatLocation(dest))
+		return nil
+	}
+	glog.V(0).Infof("uploading the current %s in place of the superseded rename", remote_storage.FormatLocation(dest))
+	remoteEntry, writeErr := retriedWriteFile(client, filerSource, dir, remoteWriteEntry(current, storageClass), dest)
+	if errors.Is(writeErr, errSuperseded) {
+		glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
+		return nil
+	}
+	if writeErr != nil {
+		return writeErr
+	}
+	return updateLocalEntry(filerClient, dir, current, remoteEntry)
+}
+
+// copyRenameInSource gives the destination key the content of a remote-only
+// entry a rename carried into the mount. The object sits at the key the
+// renamed-from path maps to on its own mount's remote, or where a
+// pending-rename marker under this mount points. The source object is left
+// alone: the source side removes it through its own delete event. When the
+// destination already holds what the stamp describes (an earlier run copied
+// it), only the stamp is refreshed. With no object on either side the event
+// fails and holds the offset.
+func copyRenameInSource(
+	filerClient filer_pb.FilerClient,
+	client remote_storage.RemoteStorageClient,
+	mountedDir string,
+	remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
+	dir string,
+	current *filer_pb.Entry,
+	renameSrc util.FullPath,
+	dest *remote_pb.RemoteStorageLocation,
+	storageClass string,
+) error {
+	srcClient, srcLoc, err := renameInSource(filerClient, client, mountedDir, remoteStorageMountLocation, current, renameSrc)
+	if err != nil {
+		return err
+	}
+	if existing, err := client.StatFile(dest); err == nil {
+		if remote_storage.Describes(current.RemoteEntry, existing) {
+			glog.V(0).Infof("%s already holds the renamed content", remote_storage.FormatLocation(dest))
+			return updateLocalEntry(filerClient, dir, current, existing)
+		}
+		glog.V(0).Infof("%s holds an object the entry does not describe (size %d, want %d); replacing it from %s", remote_storage.FormatLocation(dest), existing.RemoteSize, current.RemoteEntry.RemoteSize, remote_storage.FormatLocation(srcLoc))
+	} else if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
+		return err
+	}
+	stat, err := srcClient.StatFile(srcLoc)
+	if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
+		return fmt.Errorf("%s: content is on neither %s nor %s", util.NewFullPath(dir, current.Name), remote_storage.FormatLocation(srcLoc), remote_storage.FormatLocation(dest))
+	}
+	if err != nil {
+		return err
+	}
+	reader, err := openRemoteObject(srcClient, srcLoc, stat.RemoteSize)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+	glog.V(0).Infof("copy %s -> %s", remote_storage.FormatLocation(srcLoc), remote_storage.FormatLocation(dest))
+	remoteEntry, err := client.WriteFile(dest, remoteWriteEntry(current, storageClass), reader)
+	if err != nil {
+		return err
+	}
+	return updateLocalEntry(filerClient, dir, current, remoteEntry)
+}
+
+// renameInSource locates the object holding a remote-only rename-in entry's
+// content and the client that serves it. A pending-rename marker under this
+// mount names a key on this remote; otherwise the renamed-from path is
+// resolved through the filer's mount mappings to its own mount's remote
+// location, served by this client when the source shares this storage.
+func renameInSource(
+	filerClient filer_pb.FilerClient,
+	client remote_storage.RemoteStorageClient,
+	mountedDir string,
+	remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
+	current *filer_pb.Entry,
+	renameSrc util.FullPath,
+) (remote_storage.RemoteStorageClient, *remote_pb.RemoteStorageLocation, error) {
+	if src := util.FullPath(current.Extended[filer.ExtRemoteRenameSourceKey]); strings.HasPrefix(string(src), strings.TrimSuffix(mountedDir, "/")+"/") {
+		return client, toRemoteStorageLocation(util.FullPath(mountedDir), src, remoteStorageMountLocation), nil
+	}
+	mappings, err := readMountMappings(filerClient)
+	if err != nil {
+		return nil, nil, err
+	}
+	srcMount, srcRemoteLoc, err := filer.FindMountedRemoteMapping(mappings, string(renameSrc))
+	if err != nil {
+		return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err)
+	}
+	srcLoc := filer.MapFullPathToRemoteStorageLocation(util.FullPath(srcMount), srcRemoteLoc, renameSrc)
+	if srcRemoteLoc.Name == remoteStorageMountLocation.Name {
+		return client, srcLoc, nil
+	}
+	conf, err := readRemoteStorageConf(filerClient, srcRemoteLoc.Name)
+	if err != nil {
+		return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err)
+	}
+	srcClient, err := remote_storage.GetRemoteStorage(conf)
+	if err != nil {
+		return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err)
+	}
+	return srcClient, srcLoc, nil
+}
+
+// readMountMappings reads the filer's remote mount table, the file
+// DetectMountInfo reads at startup, through the event's filer client.
+func readMountMappings(filerClient filer_pb.FilerClient) (mappings *remote_pb.RemoteStorageMapping, err error) {
+	var content []byte
+	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		var readErr error
+		content, readErr = filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE)
+		return readErr
+	})
+	if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return nil, err
+	}
+	return filer.UnmarshalRemoteStorageMappings(content)
+}
+
+// readRemoteStorageConf reads a remote storage's configuration entry.
+func readRemoteStorageConf(filerClient filer_pb.FilerClient, storageName string) (*remote_pb.RemoteConf, error) {
+	var content []byte
+	if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		var readErr error
+		content, readErr = filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX)
+		return readErr
+	}); err != nil {
+		return nil, err
+	}
+	conf := &remote_pb.RemoteConf{}
+	if err := proto.Unmarshal(content, conf); err != nil {
+		return nil, fmt.Errorf("unmarshal %s/%s: %w", filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX, err)
+	}
+	return conf, nil
+}
+
 // isSuperseded reports whether the filer has moved past the entry an event
 // described: it is deleted, or it no longer references every chunk the event
 // named. Those are the chunks the filer deletes when an entry is updated, so