From 103328e70fcec906e7c73637db2b2d7477d8bed2 Mon Sep 17 00:00:00 2001 From: gflatters <56309061+gflatters@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:23:33 +0100 Subject: [PATCH] filer: create parent directories for lazily fetched remote entries (#11727) * filer: create parent directories for lazily fetched remote entries A remote-mounted object first read through the filer was persisted without its parent directories, so remote.uncache and the remote cache evictor, which walk down from the mount directory, never reached it: its cached chunks could not be evicted. Persist the missing parents with the entry. With those directories in place, the empty-folder cleaner leaves folders under a remote mount alone: deleting one removes its whole prefix on the remote, which can still hold objects that were never read locally. Co-Authored-By: Claude Opus 5.5 * filer: keep remote objects at parent keys and remote prefixes of empty folders Look up a lazily fetched entry's missing parents through lazy fetch before creating them, so a remote object stored at a parent's key is materialized and promoted to a prefix object rather than hidden behind a new directory. Deleting a folder that is empty in the filer is metadata-only under a remote mount, both for a non-recursive delete and for the expiry cleanup of empty parents: the mount is populated lazily, so the folder can still hold remote objects never read, and removing its remote directory deletes every object under that prefix. This replaces the empty-folder cleaner's own guard, which covered only one of those paths. A metadata-only delete records no remote-deletion tombstone, locally or from its metadata event: nothing is owed to the remote, so lazy reads stay open. Co-Authored-By: Claude Opus 5.5 * filer: stop the lazy parent lookup at a store error Only a parent the store reports as not found is missing; a parent it cannot read ends the walk, so the lookup does not stat ancestors above it on the remote. Co-Authored-By: Claude Opus 5.5 * filer: stat each missing remote parent once A lazy fetch that finds k missing parents asked FindEntry for each, and every nested fetch re-walked its own still-missing ancestors, re-stating remote keys already proven absent: 1+k(k+1)/2 remote stats for one first read. The flagged parent lookups now skip their own walk; their entries are created by the same CreateEntry that already persists the chain. * filer: materialize parents for lazily listed remote entries maybeLazyListFromRemote persisted children and the listing directory with skipCreateParentDir, so a first access that goes through the list path still left entries unreachable to a walk from the mount directory. Materialize the missing ancestors the same way the lazy fetch does, and let the synced_at directory create its parent chain. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- weed/filer/filer.go | 2 +- weed/filer/filer_delete_entry.go | 35 +++- weed/filer/filer_lazy_remote.go | 32 +++- weed/filer/filer_lazy_remote_listing.go | 6 +- weed/filer/filer_lazy_remote_test.go | 203 ++++++++++++++++++++++ weed/filer/filer_remote_tombstone.go | 2 +- weed/filer/filer_remote_tombstone_test.go | 31 ++++ 7 files changed, 302 insertions(+), 9 deletions(-) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 0a5d3540b..249ad4acd 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -890,7 +890,7 @@ func (f *Filer) DeleteEmptyParentDirectories(ctx context.Context, dirPath util.F glog.V(2).InfofCtx(ctx, "DeleteEmptyParentDirectories: deleting empty directory %s", dirPath) parentDir, _ := dirPath.DirAndName() if dirEntry, findErr := f.FindEntry(ctx, dirPath); findErr == nil { - if delErr := f.doDeleteEntryMetaAndData(ctx, dirEntry, false, false, nil); delErr == nil { + if delErr := f.doDeleteEntryMetaAndData(f.emptyFolderDeleteContext(ctx, dirPath), dirEntry, false, false, nil); delErr == nil { // Successfully deleted, continue checking upwards f.DeleteEmptyParentDirectories(ctx, util.FullPath(parentDir), stopAtPath) } else { diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 5aca5566a..bec4d7f68 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -80,6 +80,27 @@ func keepsRemoteObject(ctx context.Context) bool { return ctx.Value(keepRemoteObjectKey{}) != nil } +// emptyFolderDeleteContext makes deleting a folder that is empty in the filer +// metadata-only when it is under a remote mount. A mount is populated lazily, +// so the folder can still hold remote objects never read, and removing its +// remote directory deletes every object under that prefix. +func (f *Filer) emptyFolderDeleteContext(ctx context.Context, dirPath util.FullPath) context.Context { + if f.IsUnderRemoteMount(dirPath) { + return WithKeepRemoteObject(ctx) + } + return ctx +} + +// IsUnderRemoteMount reports whether p is a remote mount directory or lies under one. +func (f *Filer) IsUnderRemoteMount(p util.FullPath) bool { + if f.RemoteStorage == nil { + return false + } + // Mount rules are keyed with a trailing slash, so this also matches the mount directory itself. + _, remoteLoc := f.RemoteStorage.FindMountDirectory(util.FullPath(string(p) + "/")) + return remoteLoc != nil +} + func deleteEventEntry(ctx context.Context, entry *Entry) *Entry { if !keepsRemoteObject(ctx) { return entry @@ -108,6 +129,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR if ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter { return nil } + if !isRecursive && entry.IsDirectory() { + ctx = f.emptyFolderDeleteContext(ctx, p) + } isDeleteCollection := f.IsBucket(entry) collectionName := "" if isDeleteCollection { @@ -170,7 +194,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) { var dirTombstoneTs int64 - if isRecursive { + if isRecursive && !keepsRemoteObject(ctx) { // Tombstone the directory before its children: when the store drops // the subtree without listing it, or a child error aborts the walk, // the ancestor tombstone still covers every descendant. @@ -201,7 +225,9 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry for _, sub := range entries { lastFileName = sub.Name() - f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano()) + if !keepsRemoteObject(ctx) { + f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano()) + } if sub.IsDirectory() { subIsDeletingBucket := f.IsBucket(sub) err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn) @@ -264,7 +290,10 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou } } - f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano()) + // A metadata-only delete owes the remote nothing, so lazy reads stay open. + if !keepsRemoteObject(ctx) { + f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano()) + } if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil { return fmt.Errorf("filer store delete: %w", storeDeletionErr) diff --git a/weed/filer/filer_lazy_remote.go b/weed/filer/filer_lazy_remote.go index 09110adfa..036ac5f5d 100644 --- a/weed/filer/filer_lazy_remote.go +++ b/weed/filer/filer_lazy_remote.go @@ -18,11 +18,15 @@ import ( type lazyFetchContextKey struct{} +// lazyParentsContextKey marks a FindEntry issued while materializing missing +// parents: that fetch's own parent walk would only re-stat the same gaps. +type lazyParentsContextKey struct{} + // maybeLazyFetchFromRemote is called by FindEntry when the store returns no // entry for p. If p is under a remote-storage mount, it stats the remote // object, builds a filer Entry from the result, and persists it via -// CreateEntry with SkipCheckParentDirectory so phantom parent directories -// under the mount are not required. +// CreateEntry together with its missing parent directories: remote.uncache +// and cache eviction find entries by walking down from the mount directory. // // On a CreateEntry failure after a successful StatFile the in-memory entry is // still returned (availability over consistency); the singleflight key is @@ -127,7 +131,10 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) ( glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted during remote stat", p) return lazyFetchResult{nil}, nil } - saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength) + if ctx.Value(lazyParentsContextKey{}) == nil { + f.fetchMissingRemoteParents(persistBaseCtx, mountDir, p) + } + saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, false, f.MaxFilenameLength) if saveErr != nil { glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr) f.lazyFetchGroup.Forget(key) @@ -157,6 +164,25 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) ( return result.entry, nil } +// fetchMissingRemoteParents looks up p's missing parents under mountDir, top +// down, through lazy fetch. A remote object stored at a parent's key is then +// materialized first, so creating p promotes it to a prefix object instead of +// hiding it behind a plain directory. Each missing parent costs one remote +// stat, once per directory. +func (f *Filer) fetchMissingRemoteParents(ctx context.Context, mountDir, p util.FullPath) { + var missing []util.FullPath + for dir, _ := p.DirAndName(); len(dir) > len(mountDir); dir, _ = util.FullPath(dir).DirAndName() { + if _, err := f.Store.FindEntry(ctx, util.FullPath(dir)); !errors.Is(err, filer_pb.ErrNotFound) { + break + } + missing = append(missing, util.FullPath(dir)) + } + parentCtx := context.WithValue(ctx, lazyParentsContextKey{}, true) + for i := len(missing) - 1; i >= 0; i-- { + f.FindEntry(parentCtx, missing[i]) + } +} + // retractLazyRemoteEntry deletes the entry at entry.FullPath only when it is // still the entry a lazy remote read just materialized — a concurrent write // may have replaced it, and deleting by path alone would take that write down. diff --git a/weed/filer/filer_lazy_remote_listing.go b/weed/filer/filer_lazy_remote_listing.go index d692908a8..52b43981f 100644 --- a/weed/filer/filer_lazy_remote_listing.go +++ b/weed/filer/filer_lazy_remote_listing.go @@ -103,6 +103,10 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) { persistCtx := context.WithValue(context.WithoutCancel(ctx), lazyListContextKey{}, true) persistCtx = context.WithValue(persistCtx, lazyFetchContextKey{}, true) + // Entries listed under p must stay reachable by a walk from the mount + // directory, the same reason the lazy fetch fills in missing parents. + f.fetchMissingRemoteParents(context.WithoutCancel(ctx), mountDir, p) + listErr := client.ListDirectory(persistCtx, objectLoc, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { childPath := p.Child(name) @@ -215,7 +219,7 @@ func (f *Filer) updateDirectoryListingSyncedAt(ctx context.Context, p util.FullP dirEntry.Extended = make(map[string][]byte) } dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix())) - if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil { + if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, false, f.MaxFilenameLength); saveErr != nil { glog.Warningf("maybeLazyListFromRemote: create dir synced_at for %s: %v", p, saveErr) } return diff --git a/weed/filer/filer_lazy_remote_test.go b/weed/filer/filer_lazy_remote_test.go index 7a867fcc6..c56934c6f 100644 --- a/weed/filer/filer_lazy_remote_test.go +++ b/weed/filer/filer_lazy_remote_test.go @@ -17,6 +17,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/remote_storage" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/log_buffer" "github.com/seaweedfs/seaweedfs/weed/wdclient" @@ -35,6 +36,7 @@ type stubFilerStore struct { kv map[string][]byte insertErr error findErr error + findErrByPath map[string]error kvPutErr error kvGetErr error kvDeleteErr error @@ -200,6 +202,9 @@ func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry, if s.findErr != nil { return nil, s.findErr } + if err := s.findErrByPath[string(p)]; err != nil { + return nil, err + } if e, ok := s.entries[string(p)]; ok { return e, nil } @@ -340,6 +345,11 @@ func putConfigEntry(store *stubFilerStore, path string, content []byte) { } } +// putDirEntry stores a plain directory entry, so a lazy fetch below it finds its parent in place. +func putDirEntry(store *stubFilerStore, path string) { + store.entries[path] = &Entry{FullPath: util.FullPath(path), Attr: Attr{Mode: os.ModeDir | 0755}} +} + // --- tests --- func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) { @@ -375,6 +385,199 @@ func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) { assert.Equal(t, []byte("zstd"), stored.Extended["Content-Encoding"]) } +// pathStatRemoteClient stats only the objects it holds, keyed by remote path. +type pathStatRemoteClient struct { + stubRemoteClient + objects map[string]*filer_pb.RemoteEntry + statCalls []string +} + +func (c *pathStatRemoteClient) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) { + c.statCalls = append(c.statCalls, loc.Path) + if remoteEntry, found := c.objects[loc.Path]; found { + return remoteEntry, nil + } + return nil, remote_storage.ErrRemoteObjectNotFound +} + +func newPathStatFiler(t *testing.T, storageType string, client remote_storage.RemoteStorageClient) (*Filer, *stubFilerStore) { + t.Helper() + t.Cleanup(registerStubMaker(t, storageType, client)) + conf := &remote_pb.RemoteConf{Name: "mystore", Type: storageType} + rs := NewFilerRemoteStorage() + rs.storageNameToConf[conf.Name] = conf + rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{ + Name: "mystore", + Bucket: "mybucket", + Path: "/", + }) + store := newStubFilerStore() + return newTestFiler(t, store, rs), store +} + +func TestMaybeLazyFetchFromRemote_PersistsParentDirectories(t *testing.T) { + stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{ + "/a/b/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234}, + "/a/b/other.bin": {RemoteMtime: 1700000000, RemoteSize: 5678}, + }} + f, store := newPathStatFiler(t, "stub_lazy_parents", stub) + + const filePath = "/buckets/mybucket/a/b/file.bin" + entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath) + require.NoError(t, err) + require.NotNil(t, entry) + assert.Equal(t, []string{"/a/b/file.bin", "/a", "/a/b"}, stub.statCalls, "each missing parent is stat'ed once") + + for _, dir := range []string{"/buckets/mybucket/a", "/buckets/mybucket/a/b"} { + dirEntry := store.getEntry(dir) + require.NotNil(t, dirEntry, "parent %s not persisted", dir) + assert.True(t, dirEntry.IsDirectory(), "%s should be a directory", dir) + assert.Nil(t, dirEntry.Remote, "%s holds no remote object", dir) + } + + // A sibling finds its parents in place and stats only itself. + stub.statCalls = nil + _, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/other.bin") + require.NoError(t, err) + assert.Equal(t, []string{"/a/b/other.bin"}, stub.statCalls) + + // Once the content is cached, eviction walks down from the mount and must reach it. + stored := store.getEntry(filePath) + require.NotNil(t, stored) + stored.Chunks = []*filer_pb.FileChunk{{FileId: "1,01637037d6", Size: 1234}} + stored.Remote.LastLocalSyncTsNs = time.Now().UnixNano() + require.NoError(t, store.UpdateEntry(context.Background(), stored)) + + evictable := f.ListEvictableRemoteEntries(context.Background(), []util.FullPath{"/buckets/mybucket"}, 0) + require.Len(t, evictable, 1) + assert.Equal(t, util.FullPath(filePath), evictable[0].FullPath) +} + +func TestMaybeLazyFetchFromRemote_ParentLookupStopsAtStoreError(t *testing.T) { + stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{ + "/a/b/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234}, + }} + f, store := newPathStatFiler(t, "stub_lazy_parent_store_error", stub) + store.findErrByPath = map[string]error{"/buckets/mybucket/a/b": errors.New("store unavailable")} + + _, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/file.bin") + require.NoError(t, err) + assert.Equal(t, []string{"/a/b/file.bin"}, stub.statCalls, "a parent the store cannot read is not treated as missing") +} + +func TestMaybeLazyFetchFromRemote_PromotesRemoteObjectAtParentKey(t *testing.T) { + stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{ + "/reports": {RemoteMtime: 1700000000, RemoteSize: 10}, + "/reports/2026.csv": {RemoteMtime: 1700000000, RemoteSize: 20}, + }} + f, store := newPathStatFiler(t, "stub_lazy_prefix_object", stub) + + _, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/reports/2026.csv") + require.NoError(t, err) + + parent := store.getEntry("/buckets/mybucket/reports") + require.NotNil(t, parent) + assert.True(t, parent.IsDirectory()) + assert.Equal(t, []byte("true"), parent.Extended[s3_constants.SeaweedFSPrefixObject], "the remote object at the parent key must stay readable") + require.NotNil(t, parent.Remote) + assert.Equal(t, int64(10), parent.Remote.RemoteSize) +} + +func TestMaybeLazyFetchFromRemote_StatsEachParentOnce(t *testing.T) { + stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{ + "/a/b/c/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234}, + }} + f, _ := newPathStatFiler(t, "stub_lazy_parent_stats", stub) + + _, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/c/file.bin") + require.NoError(t, err) + assert.Equal(t, []string{"/a/b/c/file.bin", "/a", "/a/b", "/a/b/c"}, stub.statCalls, + "a parent still missing after its own stat must not be re-stat'ed by the next lookup") +} + +func TestMaybeLazyListFromRemote_PersistsParentDirectories(t *testing.T) { + stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{}} + stub.listDirFn = func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error { + return visitFn("/", "file.bin", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 1234}) + } + f, store := newPathStatFiler(t, "stub_lazy_list_parents", stub) + f.RemoteStorage.mapDirectoryToRemoteStorage("/buckets/listmount", &remote_pb.RemoteStorageLocation{ + Name: "mystore", + Bucket: "listmount", + Path: "/", + ListingCacheTtlSeconds: 300, + }) + + f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/listmount/a/b")) + + assert.Equal(t, []string{"/a"}, stub.statCalls, "each missing parent is stat'ed once") + for _, dir := range []string{"/buckets/listmount", "/buckets/listmount/a", "/buckets/listmount/a/b"} { + dirEntry := store.getEntry(dir) + require.NotNil(t, dirEntry, "%s not persisted", dir) + assert.True(t, dirEntry.IsDirectory(), "%s should be a directory", dir) + } + + child := store.getEntry("/buckets/listmount/a/b/file.bin") + require.NotNil(t, child) + child.Chunks = []*filer_pb.FileChunk{{FileId: "1,01637037d6", Size: 1234}} + child.Remote.LastLocalSyncTsNs = time.Now().UnixNano() + require.NoError(t, store.UpdateEntry(context.Background(), child)) + + evictable := f.ListEvictableRemoteEntries(context.Background(), []util.FullPath{"/buckets/listmount"}, 0) + require.Len(t, evictable, 1) + assert.Equal(t, util.FullPath("/buckets/listmount/a/b/file.bin"), evictable[0].FullPath) +} + +func TestDeleteEntryMetaAndData_EmptyFolderUnderMountKeepsRemotePrefix(t *testing.T) { + stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{ + "/dir/unread.bin": {RemoteMtime: 1700000000, RemoteSize: 1}, + }} + f, store := newPathStatFiler(t, "stub_lazy_empty_folder", stub) + dirPath := util.FullPath("/buckets/mybucket/dir") + putDirEntry(store, string(dirPath)) + + require.NoError(t, f.DeleteEntryMetaAndData(context.Background(), dirPath, false, false, false, false, nil, 0)) + + assert.Empty(t, stub.removeCalls, "an empty local folder must not remove its remote prefix") + assert.Nil(t, store.getEntry(string(dirPath))) + entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/unread.bin") + require.NoError(t, err) + assert.NotNil(t, entry, "a metadata-only delete must not block lazy reads under the folder") +} + +func TestDeleteEmptyParentDirectories_UnderMountKeepsRemotePrefix(t *testing.T) { + stub := &pathStatRemoteClient{} + f, store := newPathStatFiler(t, "stub_lazy_expired_parents", stub) + f.DirBucketsPath = "/buckets" + putDirEntry(store, "/buckets/mybucket/a") + putDirEntry(store, "/buckets/mybucket/a/b") + + f.DeleteEmptyParentDirectories(context.Background(), "/buckets/mybucket/a/b", "/buckets") + + assert.Empty(t, stub.removeCalls, "an empty local folder must not remove its remote prefix") + assert.Nil(t, store.getEntry("/buckets/mybucket/a/b")) + assert.Nil(t, store.getEntry("/buckets/mybucket/a")) +} + +func TestIsUnderRemoteMount(t *testing.T) { + rs := NewFilerRemoteStorage() + rs.mapDirectoryToRemoteStorage("/buckets/mirror", &remote_pb.RemoteStorageLocation{ + Name: "mystore", + Bucket: "mirror", + Path: "/", + }) + f := newTestFiler(t, newStubFilerStore(), rs) + + for path, want := range map[util.FullPath]bool{ + "/buckets/mirror": true, + "/buckets/mirror/a/b": true, + "/buckets/mirror2/a": false, + "/buckets/local/a": false, + } { + assert.Equal(t, want, f.IsUnderRemoteMount(path), path) + } +} + func TestMaybeLazyFetchFromRemote_NotUnderMount(t *testing.T) { rs := NewFilerRemoteStorage() store := newStubFilerStore() diff --git a/weed/filer/filer_remote_tombstone.go b/weed/filer/filer_remote_tombstone.go index bf7b4b187..2f4c7764d 100644 --- a/weed/filer/filer_remote_tombstone.go +++ b/weed/filer/filer_remote_tombstone.go @@ -407,7 +407,7 @@ func (f *Filer) onRemoteDeletionEvents(event *filer_pb.SubscribeMetadataResponse if message == nil { return } - if message.OldEntry != nil { + if message.OldEntry != nil && !IsMetadataOnlyDelete(message.OldEntry) { sourcePath := filer_pb.MetadataEventSourceFullPath(event) if message.NewEntry == nil || sourcePath != filer_pb.MetadataEventTargetFullPath(event) { if f.RemoteStorage != nil && f.remoteTombstones != nil { diff --git a/weed/filer/filer_remote_tombstone_test.go b/weed/filer/filer_remote_tombstone_test.go index 3d989d4c3..6048ae957 100644 --- a/weed/filer/filer_remote_tombstone_test.go +++ b/weed/filer/filer_remote_tombstone_test.go @@ -187,6 +187,7 @@ func TestMaybeLazyFetchFromRemote_SyncOffsetReleasesTombstone(t *testing.T) { }, } f, store := newMountedTestFiler(t, storageType, stub, 0) + putDirEntry(store, "/buckets/mybucket/dir") filePath := util.FullPath("/buckets/mybucket/dir/a.txt") deleteTsNs := time.Now().UnixNano() @@ -222,6 +223,7 @@ func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) { }, } f, store := newMountedTestFiler(t, storageType, stub, 0) + putDirEntry(store, "/buckets/mybucket/dir") filePath := util.FullPath("/buckets/mybucket/dir/a.txt") store.entries[string(filePath)] = &Entry{ @@ -257,6 +259,35 @@ func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) { assert.Equal(t, 1, stub.statCalls) } +func TestOnMetadataChangeEvent_MetadataOnlyDeleteSkipsTombstone(t *testing.T) { + const storageType = "stub_tomb_metadata_only" + stub := &countingRemoteClient{ + stubRemoteClient: stubRemoteClient{ + statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, + }, + } + f, store := newMountedTestFiler(t, storageType, stub, 0) + putDirEntry(store, "/buckets/mybucket/dir") + + f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ + Directory: "/buckets/mybucket", + TsNs: time.Now().UnixNano(), + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{ + Name: "dir", + IsDirectory: true, + Extended: map[string][]byte{ExtKeepRemoteObjectKey: []byte("true")}, + }, + }, + }) + + // nothing is owed to the remote, so objects under the folder stay readable + entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt") + require.NoError(t, err) + require.NotNil(t, entry) + assert.Equal(t, 1, stub.statCalls) +} + func TestOnMetadataChangeEvent_PeerDeleteTombstones(t *testing.T) { const storageType = "stub_tomb_peer" stub := &countingRemoteClient{