Compare commits

...
Author SHA1 Message Date
Chris Lu b160631fcd filer.remote.sync: let a create reshaped from a rename into the mount upload empty files
The reshape turns a rename into the mount into a create of the new entry,
but processCreateEvent's HasData gate still drops it: an empty file renamed
in leaves no remote object. A reshaped create now writes whatever the entry
holds -- stamped or not, empty or not -- while a remote-only entry still
skips since it holds nothing local to send.
2026-10-22 06:48:58 -07:00
jsas 11231b4ac6 Merge pull request #2 from seaweedfs/fix-11958-rename-in-stale-stamp
filer.remote.sync: upload a create reshaped from a rename into the mount
2026-10-22 06:29:36 -07:00
Chris LuandDevin 0bd5eec418 filer.remote.sync: upload a create reshaped from a rename into the mount
A file synced under the mount keeps its RemoteEntry across renames. The
rename out deletes the remote object, and the rename back in reshapes to a
create whose inherited stamp still matches the entry, so shouldSendToRemote
skipped the upload and the new key stayed empty. The reshaped create now
sends unconditionally: the stamp describes the old key, not this one.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-22 06:15:33 -07:00
Chris LuandDevin 0744b654ca filer.remote.sync: extract processCreateEvent
Moves the create branch of eachEntryFunc into a function, matching
processDeleteEvent and processUpdateEvent, so the create path can be
exercised directly in tests. No behavior change.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-22 06:15:21 -07:00
james b56f7ee321 filer.remote.sync: dispatch an update event by the side of the mount it touches instead of slicing the mount prefix off a path outside it
The daemon subscribes to its mount prefix and to /etc/remote. An update event
whose path lies outside the mount still reaches processUpdateEvent: an update
of the /etc/remote directory entry itself (which fs.meta.load of a mapping dump
re-creates) through the /etc/remote prefix, or a rename across the mount
boundary through the path on the inside. processUpdateEvent then maps both
paths to remote keys by slicing the mount prefix off, which panics with
"slice bounds out of range [20:11]" on a path shorter than the mount and names
a key in the wrong tree on a longer one. The event sits in the replay window,
so every restart dies on it and the persisted offset never moves.

Before dispatch, an update event is now mapped onto the side the mount covers:
a rename into the mount is the creation of the new entry, a rename out of it
the deletion of the old entry's object (kept when the entry is remote-only and
the object holds its only copy), an event with neither path inside is not the
mount's and is dropped, and an update with both paths inside is unchanged.
2026-10-22 05:01:34 -07:00
2 changed files with 348 additions and 52 deletions

No files matched your search

+129 -52
View File
@@ -140,59 +140,17 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
if filer_pb.IsEmpty(resp) {
return nil
}
renamedIntoMount := false
if message.OldEntry != nil && message.NewEntry != nil {
reshaped, inMount := reshapeCrossMountRename(mountedDir, resp)
if !inMount {
return nil
}
renamedIntoMount = reshaped != resp && filer_pb.IsCreate(reshaped)
resp, message = reshaped, reshaped.EventNotification
}
if filer_pb.IsCreate(resp) {
if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
return nil
}
// Propagate delete markers as deletions on the remote.
// Delete markers are zero-content version entries, so they
// would be filtered out by the HasData check below.
if isDeleteMarker(message.NewEntry) {
if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok {
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation)
return syncDeleteMarker(client, option, message, dest)
}
return nil
}
if !filer.HasData(message.NewEntry) {
return nil
}
glog.V(2).Infof("create: %+v", resp)
if !shouldSendToRemote(message.NewEntry) {
glog.V(2).Infof("skipping creating: %+v", resp)
return nil
}
// Rewrite internal versioning paths to the original S3 key
// to prevent double-versioning when central also has versioning enabled
parentPath, entryName := message.NewParentPath, message.NewEntry.Name
isRewrittenVersion := false
if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok {
glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName)
parentPath, entryName = newParent, newName
isRewrittenVersion = true
}
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation)
if message.NewEntry.IsDirectory {
glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest))
return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, *option.storageClass))
}
remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, *option.storageClass), dest)
if errors.Is(writeErr, errSuperseded) {
glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
return nil
}
if writeErr != nil {
return writeErr
}
// Skip updateLocalEntry for versioned rewrites: the logical
// object (e.g. file.xml) has no filer entry in versioned
// buckets, and stamping the internal v_* entry with a
// RemoteEntry for the logical key is semantically wrong.
// Replay is safe because S3 PutObject is idempotent.
if isRewrittenVersion {
return nil
}
return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
return processCreateEvent(option, filerSource, *option.storageClass, client, mountedDir, remoteStorageMountLocation, resp, renamedIntoMount)
}
if filer_pb.IsDelete(resp) {
return processDeleteEvent(filerSource, client, mountedDir, remoteStorageMountLocation, resp)
@@ -206,6 +164,125 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
return eachEntryFunc, nil
}
// reshapeCrossMountRename maps an update event onto the side of it the mount
// covers. The subscription delivers a rename when either of its paths lies
// under the mounted directory, and only a path under it maps to a remote
// key: toRemoteStorageLocation slices the mount prefix off, which panics on
// a shorter path and names a key in the wrong tree on a longer one. A rename
// into the mount is the creation of the new entry, a rename out of it the
// deletion of the old entry's object, unless that entry is remote-only and
// the object holds its only copy. An update with both paths inside is
// returned as is; one with neither is not the mount's (false).
func reshapeCrossMountRename(mountedDir string, resp *filer_pb.SubscribeMetadataResponse) (*filer_pb.SubscribeMetadataResponse, bool) {
message := resp.EventNotification
oldInMount := isUnderMount(mountedDir, resp.Directory, message.OldEntry.Name)
newInMount := isUnderMount(mountedDir, message.NewParentPath, message.NewEntry.Name)
switch {
case oldInMount && newInMount:
return resp, true
case newInMount:
glog.V(0).Infof("rename into mount %s: %s/%s -> %s/%s, creating", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name)
return &filer_pb.SubscribeMetadataResponse{
Directory: message.NewParentPath,
EventNotification: &filer_pb.EventNotification{NewEntry: message.NewEntry, NewParentPath: message.NewParentPath, Signatures: message.Signatures},
TsNs: resp.TsNs,
}, true
case oldInMount:
if isRemoteOnly(message.OldEntry) {
glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, keeping the remote-only object", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name)
return nil, false
}
glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, deleting", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name)
return &filer_pb.SubscribeMetadataResponse{
Directory: resp.Directory,
EventNotification: &filer_pb.EventNotification{OldEntry: message.OldEntry, Signatures: message.Signatures},
TsNs: resp.TsNs,
}, true
default:
return nil, false
}
}
// isUnderMount reports whether dir/name is the mounted directory or lies
// below it; a trailing slash on the mount flag does not change the answer.
func isUnderMount(mountedDir string, dir string, name string) bool {
mount := strings.TrimSuffix(mountedDir, "/")
path := string(util.NewFullPath(dir, name))
return path == mount || strings.HasPrefix(path, mount+"/")
}
// processCreateEvent uploads a created entry to the remote key its path maps
// to. A create reshaped from a rename into the mount (renamedIntoMount) writes
// whatever the entry holds: the stamp it inherited describes an object the
// move out deleted, so it cannot skip the upload, and an empty file still
// takes a zero-byte object. A remote-only entry still skips: it holds nothing
// local to send, and the filer refuses to rename one out of its mount, so its
// object remains under the key its path maps to.
func processCreateEvent(
filerClient filer_pb.FilerClient,
filerSource filer_pb.FilerClient,
storageClass string,
client remote_storage.RemoteStorageClient,
mountedDir string,
remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
resp *filer_pb.SubscribeMetadataResponse,
renamedIntoMount bool,
) error {
message := resp.EventNotification
if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
return nil
}
// Propagate delete markers as deletions on the remote.
// Delete markers are zero-content version entries, so they
// would be filtered out by the HasData check below.
if isDeleteMarker(message.NewEntry) {
if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok {
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation)
return syncDeleteMarker(client, filerClient, message, dest)
}
return nil
}
if !filer.HasData(message.NewEntry) && !(renamedIntoMount && !isRemoteOnly(message.NewEntry)) {
return nil
}
glog.V(2).Infof("create: %+v", resp)
if !renamedIntoMount && !shouldSendToRemote(message.NewEntry) {
glog.V(2).Infof("skipping creating: %+v", resp)
return nil
}
// Rewrite internal versioning paths to the original S3 key
// to prevent double-versioning when central also has versioning enabled
parentPath, entryName := message.NewParentPath, message.NewEntry.Name
isRewrittenVersion := false
if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok {
glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName)
parentPath, entryName = newParent, newName
isRewrittenVersion = true
}
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation)
if message.NewEntry.IsDirectory {
glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest))
return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, storageClass))
}
remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, storageClass), dest)
if errors.Is(writeErr, errSuperseded) {
glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
return nil
}
if writeErr != nil {
return writeErr
}
// Skip updateLocalEntry for versioned rewrites: the logical
// object (e.g. file.xml) has no filer entry in versioned
// buckets, and stamping the internal v_* entry with a
// RemoteEntry for the logical key is semantically wrong.
// Replay is safe because S3 PutObject is idempotent.
if isRewrittenVersion {
return nil
}
return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry)
}
// processDeleteEvent removes the remote object an entry mapped to. A remote
// object that is already absent counts as deleted: the entry was never
// uploaded (created and deleted faster than the sync ran, or a replay of an
+219
View File
@@ -2016,3 +2016,222 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) {
t.Fatalf("stamped RemoteEntry = %+v, want the entry's own stamp kept (mtime 100)", filerClient.updated.Entry.RemoteEntry)
}
}
// TestReshapeCrossMountRename covers an update event with a path outside
// the mount. The subscription delivers a rename when either path is under
// the mount, and an update of the /etc/remote directory entry itself through
// the daemon's /etc/remote prefix; processUpdateEvent then sliced the mount
// prefix off both paths, which panics on a path shorter than the mount (seen
// on a replay: "slice bounds out of range [20:11]" on the 11-byte
// /etc/remote, every restart dying on the same event) and names a key in the
// wrong tree on a longer one.
func TestReshapeCrossMountRename(t *testing.T) {
const mount = "/buckets/mount"
chunked := func(name string) *filer_pb.Entry {
return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, Chunks: []*filer_pb.FileChunk{{FileId: "3,01", Size: 7}}}
}
event := func(oldDir, oldName, newDir, newName string, oldEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
if oldEntry == nil {
oldEntry = chunked(oldName)
}
return &filer_pb.SubscribeMetadataResponse{
Directory: oldDir,
TsNs: 42,
EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewParentPath: newDir, NewEntry: chunked(newName), Signatures: []int32{7}},
}
}
t.Run("into the mount from a shorter path is a create of the new entry", func(t *testing.T) {
resp := event("/plain", "in.txt", mount+"/lock", "in.txt", nil)
got, inMount := reshapeCrossMountRename(mount, resp)
if !inMount || !filer_pb.IsCreate(got) {
t.Fatalf("got %+v inMount=%v, want a create", got, inMount)
}
if got.Directory != mount+"/lock" || got.EventNotification.NewParentPath != mount+"/lock" || got.EventNotification.NewEntry.Name != "in.txt" {
t.Errorf("create at %s/%s (dir %s), want %s/in.txt", got.EventNotification.NewParentPath, got.EventNotification.NewEntry.Name, got.Directory, mount+"/lock")
}
if got.TsNs != 42 || len(got.EventNotification.Signatures) != 1 {
t.Errorf("timestamp and signatures must carry over: %+v", got)
}
})
t.Run("into the mount from a longer path outside it is a create, not a key in the wrong tree", func(t *testing.T) {
resp := event("/buckets/other/deeper/tree", "x.txt", mount, "x.txt", nil)
got, inMount := reshapeCrossMountRename(mount, resp)
if !inMount || !filer_pb.IsCreate(got) || got.EventNotification.NewParentPath != mount {
t.Fatalf("got %+v inMount=%v, want a create under %s", got, inMount, mount)
}
})
t.Run("out of the mount is a delete of the old entry", func(t *testing.T) {
resp := event(mount+"/lock", "out.txt", "/plain", "out.txt", nil)
got, inMount := reshapeCrossMountRename(mount, resp)
if !inMount || !filer_pb.IsDelete(got) {
t.Fatalf("got %+v inMount=%v, want a delete", got, inMount)
}
if got.Directory != mount+"/lock" || got.EventNotification.OldEntry.Name != "out.txt" {
t.Errorf("delete of %s/%s, want %s/out.txt", got.Directory, got.EventNotification.OldEntry.Name, mount+"/lock")
}
})
t.Run("out of the mount keeps the object of a remote-only entry", func(t *testing.T) {
remoteOnly := &filer_pb.Entry{Name: "out.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteSize: 7, RemoteMtime: 1786096669}}
resp := event(mount+"/lock", "out.txt", "/plain", "out.txt", remoteOnly)
if got, inMount := reshapeCrossMountRename(mount, resp); inMount || got != nil {
t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount)
}
})
t.Run("both paths inside is the update itself", func(t *testing.T) {
resp := event(mount+"/a", "x.txt", mount+"/b", "y.txt", nil)
if got, inMount := reshapeCrossMountRename(mount, resp); !inMount || got != resp {
t.Fatalf("got %+v inMount=%v, want the same event", got, inMount)
}
})
t.Run("an update of the /etc/remote directory entry itself is not the mount's event", func(t *testing.T) {
dirEntry := &filer_pb.Entry{Name: "remote", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}}
resp := &filer_pb.SubscribeMetadataResponse{
Directory: "/etc",
EventNotification: &filer_pb.EventNotification{OldEntry: dirEntry, NewParentPath: "/etc", NewEntry: dirEntry},
}
if got, inMount := reshapeCrossMountRename("/buckets/c4x-storage", resp); inMount || got != nil {
t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount)
}
})
t.Run("neither path inside is not the mount's event", func(t *testing.T) {
resp := event("/plain", "x.txt", "/buckets/mounted-not", "x.txt", nil)
if got, inMount := reshapeCrossMountRename(mount, resp); inMount || got != nil {
t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount)
}
})
t.Run("a trailing slash on the mount flag changes nothing", func(t *testing.T) {
resp := event("/plain", "in.txt", mount, "in.txt", nil)
if got, inMount := reshapeCrossMountRename(mount+"/", resp); !inMount || !filer_pb.IsCreate(got) {
t.Fatalf("got %+v inMount=%v, want a create", got, inMount)
}
})
}
// TestRenameIntoMountUploadsDespiteTheOldStamp covers a file synced at a path
// under the mount, renamed out (which deleted its remote object), and renamed
// back in: the entry still carries the RemoteEntry the first sync stamped, so
// shouldSendToRemote would skip the create the reshape made, leaving the new
// key without an object. A reshaped create must send unconditionally.
func TestRenameIntoMountUploadsDespiteTheOldStamp(t *testing.T) {
const mountedDir = "/buckets/mount"
mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"}
stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1}
newEntry := &filer_pb.Entry{
Name: "back.txt",
Content: []byte("payload"),
Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669},
RemoteEntry: stamp,
}
resp, inMount := reshapeCrossMountRename(mountedDir, &filer_pb.SubscribeMetadataResponse{
Directory: "/plain",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "back.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: stamp},
NewParentPath: mountedDir + "/lock",
NewEntry: newEntry,
},
})
if !inMount || !filer_pb.IsCreate(resp) {
t.Fatalf("reshape = %+v inMount=%v, want a create", resp, inMount)
}
if shouldSendToRemote(resp.EventNotification.NewEntry) {
t.Fatal("precondition: the stamp the rename carried should fail shouldSendToRemote")
}
remote := &recordingRemote{}
filerClient := &stubFilerClient{entry: newEntry}
if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp, true); err != nil {
t.Fatal(err)
}
wantWrite := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/lock/back.txt"}
if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) {
t.Fatalf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite))
}
// a plain create keeps trusting the stamp
remote = &recordingRemote{}
if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp, false); err != nil {
t.Fatal(err)
}
if len(remote.writes) != 0 {
t.Errorf("writes = %+v, want the stamped create skipped", remote.writes)
}
}
// TestRenameIntoMountCreateUploads covers what the reshape hands the create
// path: a rename into the mount becomes a create, and that create cannot
// trust the stamp or the missing content the entry carries. An empty file
// still takes a zero-byte object on the remote; a remote-only entry holds
// nothing local to send, so it still skips.
func TestRenameIntoMountCreateUploads(t *testing.T) {
const mountedDir = "/buckets/mount"
mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"}
stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1}
renameIn := func(entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
resp := &filer_pb.SubscribeMetadataResponse{
Directory: "/data/staging",
EventNotification: &filer_pb.EventNotification{OldEntry: entry, NewParentPath: mountedDir + "/lock", NewEntry: entry},
}
reshaped, inMount := reshapeCrossMountRename(mountedDir, resp)
if !inMount || !filer_pb.IsCreate(reshaped) {
t.Fatalf("reshape got %+v inMount=%v, want a create", reshaped, inMount)
}
return reshaped
}
want := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/lock/in.txt"}
t.Run("an empty entry writes a zero-byte object", func(t *testing.T) {
entry := &filer_pb.Entry{Name: "in.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}}
if filer.HasData(entry) {
t.Fatal("precondition: an empty file carries no data")
}
remote := &recordingRemote{objects: map[string][]byte{}}
filerClient := &stubFilerClient{entry: entry}
if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true); err != nil {
t.Fatal(err)
}
if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], want) {
t.Errorf("writes = %+v, want the mounted key %s written", remote.writes, remote_storage.FormatLocation(want))
}
})
t.Run("a remote-only entry still does not upload", func(t *testing.T) {
entry := &filer_pb.Entry{Name: "in.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, RemoteEntry: stamp}
if !isRemoteOnly(entry) {
t.Fatal("precondition: entry should be remote-only")
}
remote := &recordingRemote{objects: map[string][]byte{}}
filerClient := &stubFilerClient{entry: entry}
if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true); err != nil {
t.Fatal(err)
}
if len(remote.writes) != 0 {
t.Errorf("writes = %+v, want none: there is nothing local to send", remote.writes)
}
})
t.Run("an ordinary create keeps the stamp and data gates", func(t *testing.T) {
empty := &filer_pb.Entry{Name: "in.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}}
create := &filer_pb.SubscribeMetadataResponse{
Directory: mountedDir + "/lock",
EventNotification: &filer_pb.EventNotification{NewParentPath: mountedDir + "/lock", NewEntry: empty},
}
remote := &recordingRemote{objects: map[string][]byte{}}
filerClient := &stubFilerClient{entry: empty}
if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, create, false); err != nil {
t.Fatal(err)
}
if len(remote.writes) != 0 {
t.Errorf("writes = %+v, want none: an ordinary create of an empty file still skips", remote.writes)
}
})
}