diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index 9a343d18eb..56333b4def 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -143,6 +143,25 @@ func (store *LevelDBStore) BatchInsertEntries(ctx context.Context, entries []*fi return nil } +// BatchDeleteEntries deletes multiple entries in a single LevelDB batch write, +// the counterpart of BatchInsertEntries. +func (store *LevelDBStore) BatchDeleteEntries(ctx context.Context, paths []weed_util.FullPath) error { + if len(paths) == 0 { + return nil + } + + batch := new(leveldb.Batch) + for _, fullpath := range paths { + batch.Delete(genKey(fullpath.DirAndName())) + } + + if err := store.db.Write(batch, writeOptions(ctx)); err != nil { + return fmt.Errorf("batch delete: %w", err) + } + + return nil +} + func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { key := genKey(fullpath.DirAndName()) diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index 4f9c13719e..b78198a85d 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -160,3 +161,37 @@ func TestUnavailableDatabaseFailsListAndDelete(t *testing.T) { t.Fatal("delete children on a closed database succeeded") } } + +// TestBatchDeleteEntries: one batch removes exactly the named entries, a name +// that is not stored is not an error, and an empty batch is a no-op. +func TestBatchDeleteEntries(t *testing.T) { + store := &LevelDBStore{} + store.initialize(t.TempDir()) + defer store.Shutdown() + ctx := context.Background() + + var entries []*filer.Entry + for _, p := range []util.FullPath{"/dir/a", "/dir/b", "/dir/c", "/other/a"} { + entries = append(entries, &filer.Entry{FullPath: p, Attr: filer.Attr{Mode: 0644}}) + } + if err := store.BatchInsertEntries(ctx, entries); err != nil { + t.Fatalf("batch insert: %v", err) + } + + if err := store.BatchDeleteEntries(ctx, nil); err != nil { + t.Fatalf("empty batch delete: %v", err) + } + if err := store.BatchDeleteEntries(ctx, []util.FullPath{"/dir/a", "/dir/missing", "/dir/c"}); err != nil { + t.Fatalf("batch delete: %v", err) + } + + for p, wantFound := range map[util.FullPath]bool{"/dir/a": false, "/dir/b": true, "/dir/c": false, "/other/a": true} { + _, err := store.FindEntry(ctx, p) + if wantFound && err != nil { + t.Errorf("%s should have been kept: %v", p, err) + } + if !wantFound && err != filer_pb.ErrNotFound { + t.Errorf("%s should have been deleted, FindEntry returned %v", p, err) + } + } +} diff --git a/weed/mount/meta_cache/meta_cache.go b/weed/mount/meta_cache/meta_cache.go index e4c16979df..e044a5c576 100644 --- a/weed/mount/meta_cache/meta_cache.go +++ b/weed/mount/meta_cache/meta_cache.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "errors" - "math" "os" "sync" "time" @@ -49,6 +48,10 @@ type MetaCache struct { // instead of streaming to the limit again to rediscover them. oversizedDirs map[util.FullPath]struct{} + // rebuildWipeInserts holds the children inserted into each directory + // while its rebuild wipe runs; the wipe keeps them. + rebuildWipeInserts map[util.FullPath]map[string]struct{} + // dirVersionFloors is each cached directory's listing snapshot: the // version of every child the listing covered, present or absent, unless // a later event gave that child its own record. One map write per build @@ -86,6 +89,9 @@ var ( type directoryBuildState struct { bufferedEvents []*filer_pb.SubscribeMetadataResponse + // removals the rebuild wipe declined as being of another file than the + // one cached; the replay checks them again against what it finds + declinedRemovals map[*filer_pb.SubscribeMetadataResponse]struct{} } const recentEventDedupWindow = 4096 @@ -135,6 +141,7 @@ func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPat dirVersionFloors: make(map[util.FullPath]int64), dirSections: make(map[util.FullPath]*sectionList), oversizedDirs: make(map[util.FullPath]struct{}), + rebuildWipeInserts: make(map[util.FullPath]map[string]struct{}), } mc.invalidateWorker = util.NewAsyncBatchWorker(func(batch []EntryInvalidation) { for _, invalidation := range batch { @@ -176,6 +183,10 @@ func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry, vers if err := mc.localStore.InsertEntry(ctx, entry); err != nil { return err } + dir, name := entry.FullPath.DirAndName() + if inserted, wiping := mc.rebuildWipeInserts[util.FullPath(dir)]; wiping { + inserted[name] = struct{}{} + } mc.setEntryVersionLocked(ctx, entry.FullPath, versionTsNs) mc.noteHardLinkVersionLocked(ctx, entry.HardLinkId, versionTsNs) return nil @@ -718,34 +729,87 @@ func (mc *MetaCache) SetPinnedChildFn(fn func(*filer.Entry) bool) { mc.pinnedChildFn = fn } +// rebuildWipeBatchSize caps how many cached children one locked step of a +// rebuild's wipe visits. Every metadata read in the mount takes mc's read lock, +// and a pending writer blocks new readers, so a wipe that held the write lock +// across a whole directory stalled lookups in unrelated directories for as long +// as the scan took — longer still while leveldb carries the deletion markers of +// earlier wipes of the same directory. +const rebuildWipeBatchSize = 1024 + // deleteFolderChildrenForRebuild clears a directory's cached children ahead of a // rebuild, but keeps any child flagged pinned by pinnedChildFn — a local-only // create not yet flushed to the filer. A rebuild refills from a filer listing // that does not include such a create; a blind wipe would drop it and then // markCachedFn publishes the directory authoritatively cached without a file the // client created, so it vanishes from the mount. +// +// The wipe runs in batches and lets go of the lock between them, so what the +// single-lock wipe made wait now lands while it runs: a create's placeholder, +// which keepInsertsDuringWipe keeps, and the unlink or rename that ends such a +// file before the listing starts, which vacateDuringWipe applies. func (mc *MetaCache) deleteFolderChildrenForRebuild(ctx context.Context, dirPath util.FullPath) error { + defer mc.keepInsertsDuringWipe(dirPath)() + startFrom := "" + for { + last, err := mc.deleteFolderChildrenBatch(ctx, dirPath, startFrom) + if err != nil { + return err + } + if last == "" { + return nil + } + startFrom = last + } +} + +// keepInsertsDuringWipe makes a wipe of dirPath keep the children inserted +// until the returned func runs. A deferred create inserts its placeholder +// before its handle pins it; the single-lock wipe kept such an insert by +// making it wait, and a batch running in that gap must not drop it either. +func (mc *MetaCache) keepInsertsDuringWipe(dirPath util.FullPath) func() { + mc.Lock() + mc.rebuildWipeInserts[dirPath] = make(map[string]struct{}) + mc.Unlock() + return func() { + mc.Lock() + delete(mc.rebuildWipeInserts, dirPath) + mc.Unlock() + } +} + +// deleteFolderChildrenBatch deletes the unpinned ones among the next +// rebuildWipeBatchSize children of dirPath that sort after startFrom. It +// returns the name to resume from, or "" once the directory is exhausted. A +// child is judged pinned under the same lock that deletes it, so a create +// pinned between batches survives as surely as one pinned before the wipe +// began. +func (mc *MetaCache) deleteFolderChildrenBatch(ctx context.Context, dirPath util.FullPath, startFrom string) (string, error) { mc.Lock() defer mc.Unlock() - if mc.pinnedChildFn == nil { - return mc.localStore.DeleteFolderChildren(ctx, dirPath) - } - var pinned []*filer.Entry - if _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, "", true, math.MaxInt64, func(entry *filer.Entry) (bool, error) { - if mc.pinnedChildFn(entry) { - pinned = append(pinned, entry) + visited := 0 + var doomed []util.FullPath + inserted := mc.rebuildWipeInserts[dirPath] + last, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFrom, false, rebuildWipeBatchSize, func(entry *filer.Entry) (bool, error) { + visited++ + if _, kept := inserted[entry.Name()]; kept { + return true, nil + } + if mc.pinnedChildFn == nil || !mc.pinnedChildFn(entry) { + doomed = append(doomed, entry.FullPath) } return true, nil - }); err != nil { - return err + }) + if err != nil { + return "", err } - if err := mc.localStore.DeleteFolderChildren(ctx, dirPath); err != nil { - return err + if err := mc.leveldbStore.BatchDeleteEntries(ctx, doomed); err != nil { + return "", err } - if len(pinned) > 0 { - return mc.doBatchInsertEntries(ctx, pinned) + if visited < rebuildWipeBatchSize { + return "", nil } - return nil + return last, nil } // ListDirectoryEntries reports the last name the store reached, which is not @@ -973,10 +1037,102 @@ func (mc *MetaCache) applyMetadataResponseNow(ctx context.Context, resp *filer_p continue } state.bufferedEvents = append(state.bufferedEvents, events...) + for _, declined := range mc.vacateDuringWipe(ctx, events) { + if state.declinedRemovals == nil { + state.declinedRemovals = make(map[*filer_pb.SubscribeMetadataResponse]struct{}) + } + state.declinedRemovals[declined] = struct{}{} + } } return nil } +// vacateDuringWipe takes out of the store, at once, the paths that events +// buffered for a directory vacate while its rebuild wipe runs. The build +// replays only the events later than its listing's snapshot and leaves the +// earlier ones to the listing, which starts after the wipe. A listing shows +// that a file is gone only by not having it, and that removes nothing: an +// entry the wipe kept, a deferred create's placeholder, would stay cached +// after its file was flushed and then unlinked or renamed away before the +// wipe ended. Once the wipe is over the store holds what the listing put +// there, and a removal earlier than the snapshot may still arrive, so the +// event is left to the replay again. +// +// It returns the removals it declined as being of another file, which an +// unversioned listing would otherwise replay over the entry the wipe kept. +func (mc *MetaCache) vacateDuringWipe(ctx context.Context, events []*filer_pb.SubscribeMetadataResponse) (declined []*filer_pb.SubscribeMetadataResponse) { + mc.Lock() + defer mc.Unlock() + for _, event := range events { + oldDir, hasOld := metadataOldParentDir(event) + if !hasOld { + continue + } + if _, wiping := mc.rebuildWipeInserts[oldDir]; !wiping { + continue + } + old := event.EventNotification.OldEntry + oldPath := oldDir.Child(old.Name) + if newDir, hasNew := metadataNewParentDir(event); hasNew && newDir.Child(event.EventNotification.NewEntry.Name) == oldPath { + continue // rewritten in place + } + if event.TsNs != 0 && mc.entryVersionBlocksLocked(ctx, oldPath, event.TsNs, true) { + continue // older than what is cached there + } + // A placeholder is unversioned and fenced against nothing, so a late + // removal of an earlier file of the same name is told apart by the + // file it took away. + cached, err := mc.localStore.FindEntry(ctx, oldPath) + if err != nil || cached == nil { + continue + } + if !isCachedFile(old, cached) { + declined = append(declined, event) + continue + } + // OP=MV, as where a replayed event vacates a path: taking a name out + // of the cache is not an unlink of a hard link's shared record. + if err := mc.localStore.DeleteEntry(context.WithValue(ctx, "OP", "MV"), oldPath); err != nil { + glog.V(1).Infof("vacate %s while its directory is wiped: %v", oldPath, err) + continue + } + // The placeholder's record goes with it, or the name would stay + // unversioned. + mc.clearEntryVersionLocked(ctx, oldPath) + } + return declined +} + +// declinedRemovalForReplay returns the removal the wipe declined, or only the +// name it puts in place if the entry the replay finds there is still another +// file. Earlier buffered events may have replaced the one the wipe saw. +func (mc *MetaCache) declinedRemovalForReplay(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) *filer_pb.SubscribeMetadataResponse { + mc.Lock() + defer mc.Unlock() + oldDir, _ := metadataOldParentDir(event) + old := event.EventNotification.OldEntry + cached, err := mc.localStore.FindEntry(ctx, oldDir.Child(old.Name)) + if err != nil || cached == nil || isCachedFile(old, cached) { + return event + } + return metadataCreateFragment(event) +} + +// isCachedFile reports whether old, the entry an event took away, is the file +// cached at its path: the same inode, or with no inode to compare, the same +// creation time. An event that only names the file cannot tell, and is taken +// at its word. +func isCachedFile(old *filer_pb.Entry, cached *filer.Entry) bool { + attributes := old.GetAttributes() + if attributes == nil { + return true + } + if attributes.Inode != 0 && cached.Attr.Inode != 0 { + return attributes.Inode == cached.Attr.Inode + } + return attributes.Crtime == 0 || attributes.Crtime == cached.Attr.Crtime.Unix() +} + func (mc *MetaCache) applyMetadataResponseDirect(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions, allowUncachedInsert bool) error { if _, err := mc.applyMetadataResponseLocked(ctx, resp, options, allowUncachedInsert); err != nil { return err @@ -1171,7 +1327,15 @@ func (mc *MetaCache) completeDirectoryBuildNow(ctx context.Context, dirPath util } mc.Unlock() - for _, event := range state.bufferedEvents { + for i, event := range state.bufferedEvents { + // Decided here for the invalidations below too. + if _, declined := state.declinedRemovals[event]; declined { + event = mc.declinedRemovalForReplay(ctx, event) + state.bufferedEvents[i] = event + if event == nil { + continue + } + } // When the server provided a snapshot timestamp, skip events that // the listing already included. When snapshotTsNs == 0 (empty // directory — server returned no entries and no snapshot), replay @@ -1191,6 +1355,9 @@ func (mc *MetaCache) completeDirectoryBuildNow(ctx context.Context, dirPath util // so a handle can hold older state than the completed directory. After // markCachedFn, versioned at the snapshot to outrank the mid-build install. for _, event := range state.bufferedEvents { + if event == nil { + continue + } if event.TsNs < snapshotTsNs { event.TsNs = snapshotTsNs } @@ -1273,7 +1440,8 @@ func metadataDeleteFragment(resp *filer_pb.SubscribeMetadataResponse) *filer_pb. return &filer_pb.SubscribeMetadataResponse{ Directory: resp.Directory, EventNotification: &filer_pb.EventNotification{ - OldEntry: proto.Clone(resp.EventNotification.OldEntry).(*filer_pb.Entry), + OldEntry: proto.Clone(resp.EventNotification.OldEntry).(*filer_pb.Entry), + Signatures: resp.EventNotification.Signatures, }, TsNs: resp.TsNs, } @@ -1292,6 +1460,7 @@ func metadataCreateFragment(resp *filer_pb.SubscribeMetadataResponse) *filer_pb. EventNotification: &filer_pb.EventNotification{ NewEntry: proto.Clone(resp.EventNotification.NewEntry).(*filer_pb.Entry), NewParentPath: newDir, + Signatures: resp.EventNotification.Signatures, }, TsNs: resp.TsNs, } diff --git a/weed/mount/meta_cache/meta_cache_apply_test.go b/weed/mount/meta_cache/meta_cache_apply_test.go index cee1da0e23..2e90ea6aaf 100644 --- a/weed/mount/meta_cache/meta_cache_apply_test.go +++ b/weed/mount/meta_cache/meta_cache_apply_test.go @@ -473,6 +473,34 @@ func TestCollectEntryInvalidationsCarryAuthoritativeEntries(t *testing.T) { } } +// TestRenameFragmentsKeepSignatures: a handle fences an invalidation by its +// timestamp only within the filer clock the signatures name, so the halves a +// rename is split into keep the event's. +func TestRenameFragmentsKeepSignatures(t *testing.T) { + rename := &filer_pb.SubscribeMetadataResponse{ + Directory: "/src", + TsNs: 78, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file.tmp"}, + NewEntry: &filer_pb.Entry{Name: "file.txt"}, + NewParentPath: "/dst", + Signatures: []int32{7}, + }, + } + var got []EntryInvalidation + for _, fragment := range []*filer_pb.SubscribeMetadataResponse{metadataDeleteFragment(rename), metadataCreateFragment(rename)} { + got = append(got, collectEntryInvalidations(fragment)...) + } + if len(got) != 2 { + t.Fatalf("fragment invalidations = %+v, want one per half", got) + } + for _, invalidation := range got { + if len(invalidation.Signatures) != 1 || invalidation.Signatures[0] != 7 { + t.Errorf("invalidation of %s carries signatures %v, want [7]", invalidation.Path, invalidation.Signatures) + } + } +} + func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) { t.Helper() diff --git a/weed/mount/meta_cache/meta_cache_rebuild_wipe_test.go b/weed/mount/meta_cache/meta_cache_rebuild_wipe_test.go new file mode 100644 index 0000000000..788e11d1e7 --- /dev/null +++ b/weed/mount/meta_cache/meta_cache_rebuild_wipe_test.go @@ -0,0 +1,612 @@ +package meta_cache + +import ( + "context" + "fmt" + "math" + "reflect" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func rebuildWipeChildName(i int) string { + return fmt.Sprintf("f%06d", i) +} + +// fillRebuildWipeDir caches n children of dir, named by rebuildWipeChildName. +func fillRebuildWipeDir(t *testing.T, mc *MetaCache, dir util.FullPath, n int) { + t.Helper() + entries := make([]*filer.Entry, 0, n) + for i := 0; i < n; i++ { + entries = append(entries, &filer.Entry{ + FullPath: dir.Child(rebuildWipeChildName(i)), + Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1}, + }) + } + if err := mc.doBatchInsertEntries(context.Background(), entries); err != nil { + t.Fatalf("fill %s: %v", dir, err) + } +} + +// insertPinnedCacheEntry caches an entry the tests' pinnedChildFn keeps. +func insertPinnedCacheEntry(t *testing.T, mc *MetaCache, path util.FullPath) { + t.Helper() + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: path, + Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 42}, + }, 0); err != nil { + t.Fatalf("insert pinned %s: %v", path, err) + } +} + +func pinInode42(e *filer.Entry) bool { return e.Attr.Inode == 42 } + +func cachedChildNames(t *testing.T, mc *MetaCache, dir util.FullPath) []string { + t.Helper() + var names []string + if _, err := mc.localStore.ListDirectoryEntries(context.Background(), dir, "", true, math.MaxInt64, func(entry *filer.Entry) (bool, error) { + names = append(names, entry.Name()) + return true, nil + }); err != nil { + t.Fatalf("list %s: %v", dir, err) + } + return names +} + +// TestRebuildWipeSpansBatches: a wipe that takes several batches still drops +// every unpinned child, keeps a pinned one in whichever batch it falls, and +// leaves everything that is not a direct child alone. +func TestRebuildWipeSpansBatches(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + const n = 2*rebuildWipeBatchSize + 10 + fillRebuildWipeDir(t, mc, "/dir", n) + pinnedFirst := rebuildWipeChildName(5) + "-pending" + pinnedLast := rebuildWipeChildName(n-3) + "-pending" + insertPinnedCacheEntry(t, mc, util.FullPath("/dir").Child(pinnedFirst)) + insertPinnedCacheEntry(t, mc, util.FullPath("/dir").Child(pinnedLast)) + insertCacheEntry(t, mc, "/dir/sub/grandchild") + insertCacheEntry(t, mc, "/other/keep") + + if err := mc.deleteFolderChildrenForRebuild(context.Background(), "/dir"); err != nil { + t.Fatalf("wipe: %v", err) + } + + if got, want := cachedChildNames(t, mc, "/dir"), []string{pinnedFirst, pinnedLast}; !reflect.DeepEqual(got, want) { + t.Errorf("children left by the wipe = %v, want only the pinned %v", got, want) + } + for _, p := range []util.FullPath{"/dir/sub/grandchild", "/other/keep"} { + if _, _, err := mc.FindEntry(context.Background(), p); err != nil { + t.Errorf("%s is not a child of the wiped directory but was lost: %v", p, err) + } + } +} + +// TestRebuildWipeBatchIsBoundedAndJudgesPinnedLate drives the wipe one batch at +// a time. One batch removes at most rebuildWipeBatchSize children and leaves +// the lock free; what lands between batches is judged when its batch runs: a +// local-only create is kept, a stale entry is wiped. +func TestRebuildWipeBatchIsBoundedAndJudgesPinnedLate(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + const n = 2*rebuildWipeBatchSize + 10 + fillRebuildWipeDir(t, mc, "/dir", n) + + last, err := mc.deleteFolderChildrenBatch(context.Background(), "/dir", "") + if err != nil { + t.Fatalf("first batch: %v", err) + } + if !mc.TryRLock() { + t.Fatal("a wipe batch returned with the meta cache still locked") + } + mc.RUnlock() + if want := rebuildWipeChildName(rebuildWipeBatchSize - 1); last != want { + t.Fatalf("first batch resumes after %q, want %q", last, want) + } + if got := len(cachedChildNames(t, mc, "/dir")); got != n-rebuildWipeBatchSize { + t.Fatalf("children left after one batch = %d, want %d", got, n-rebuildWipeBatchSize) + } + + // Both sort after every child the wipe has yet to visit. + insertPinnedCacheEntry(t, mc, "/dir/zz-pending") + insertCacheEntry(t, mc, "/dir/zz-stale") + + batches := 1 + for last != "" { + if last, err = mc.deleteFolderChildrenBatch(context.Background(), "/dir", last); err != nil { + t.Fatalf("batch %d: %v", batches+1, err) + } + batches++ + } + if batches != 3 { + t.Errorf("wipe of %d children took %d batches, want 3", n+2, batches) + } + if got, want := cachedChildNames(t, mc, "/dir"), []string{"zz-pending"}; !reflect.DeepEqual(got, want) { + t.Errorf("children left by the wipe = %v, want only the create pinned between batches %v", got, want) + } +} + +// rebuildWipeReader reads another directory under mc's read lock and reports, +// while it still holds the lock, that it got in. +func rebuildWipeReader(mc *MetaCache, gotIn *atomic.Bool, done chan<- struct{}) { + defer close(done) + mc.ListDirectoryEntries(context.Background(), "/other", "", true, 1, func(*filer.Entry) (bool, error) { + gotIn.Store(true) + return true, nil + }) +} + +// waitRebuildWipeReaderParked returns once rebuildWipeReader is parked on mc's +// read lock. The stack is a witness of where the reader is, not a timing +// assertion: no sleep decides who wins, and the deadline only refuses a broken +// arrangement. +func waitRebuildWipeReaderParked(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + buf := make([]byte, 1<<20) + for time.Now().Before(deadline) { + n := runtime.Stack(buf, true) + for n == len(buf) { + buf = make([]byte, 2*len(buf)) + n = runtime.Stack(buf, true) + } + for _, stack := range strings.Split(string(buf[:n]), "\n\n") { + header, _, _ := strings.Cut(stack, "\n") + if strings.Contains(header, "RWMutex.RLock") && strings.Contains(stack, ".rebuildWipeReader(") { + return + } + } + runtime.Gosched() + } + t.Fatal("the reader never parked on the meta cache read lock") +} + +// TestRebuildWipeLetsReadersInBetweenBatches: a read of an unrelated directory +// that is parked behind the wipe gets the lock when the first batch ends, +// instead of waiting for the whole directory to be scanned. +func TestRebuildWipeLetsReadersInBetweenBatches(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/other": true}) + defer mc.Shutdown() + + fillRebuildWipeDir(t, mc, "/dir", 2*rebuildWipeBatchSize+1) + insertCacheEntry(t, mc, "/other/x") + + var gotIn atomic.Bool + readDone := make(chan struct{}) + calls := 0 + gotInBySecondBatch := false + // pinnedChildFn runs once per child on this goroutine, under the wipe's + // write lock. + mc.SetPinnedChildFn(func(*filer.Entry) bool { + calls++ + switch calls { + case 1: + go rebuildWipeReader(mc, &gotIn, readDone) + waitRebuildWipeReaderParked(t) + case rebuildWipeBatchSize + 1: + // First child of the second batch. A reader parked when the first + // batch unlocked holds the lock before the next writer gets it, + // and the reader reports from under the lock. + gotInBySecondBatch = gotIn.Load() + } + return false + }) + + if err := mc.deleteFolderChildrenForRebuild(context.Background(), "/dir"); err != nil { + t.Fatalf("wipe: %v", err) + } + select { + case <-readDone: + case <-time.After(10 * time.Second): + t.Fatal("the reader never returned") + } + if calls != 2*rebuildWipeBatchSize+1 { + t.Fatalf("pinnedChildFn calls = %d, want %d", calls, 2*rebuildWipeBatchSize+1) + } + if !gotInBySecondBatch { + t.Fatal("a reader parked behind the wipe waited past the first batch; the wipe should release the lock between batches") + } +} + +// TestRebuildWipeKeepsInsertsWhileItRuns: a deferred create's placeholder can +// land between batches before its handle pins it. The wipe keeps it, as the +// single-lock wipe did by making the insert wait. +func TestRebuildWipeKeepsInsertsWhileItRuns(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + const n = 2*rebuildWipeBatchSize + 10 + fillRebuildWipeDir(t, mc, "/dir", n) + + stopKeeping := mc.keepInsertsDuringWipe("/dir") + last, err := mc.deleteFolderChildrenBatch(context.Background(), "/dir", "") + if err != nil { + t.Fatalf("first batch: %v", err) + } + insertCacheEntry(t, mc, "/dir/zz-placeholder") + for batches := 1; last != ""; batches++ { + if last, err = mc.deleteFolderChildrenBatch(context.Background(), "/dir", last); err != nil { + t.Fatalf("batch %d: %v", batches+1, err) + } + } + stopKeeping() + + if got, want := cachedChildNames(t, mc, "/dir"), []string{"zz-placeholder"}; !reflect.DeepEqual(got, want) { + t.Errorf("children left by the wipe = %v, want only the one inserted while it ran %v", got, want) + } + if len(mc.rebuildWipeInserts) != 0 { + t.Errorf("rebuildWipeInserts = %v after the wipe ended, want empty", mc.rebuildWipeInserts) + } +} + +// midRebuildWipe begins a build of /dir over more children than one batch +// holds and runs the wipe's first batch. The returned func runs the rest of +// the wipe. +func midRebuildWipe(t *testing.T, mc *MetaCache) (finishWipe func()) { + t.Helper() + ctx := context.Background() + fillRebuildWipeDir(t, mc, "/dir", 2*rebuildWipeBatchSize+10) + if err := mc.BeginDirectoryBuild(ctx, "/dir"); err != nil { + t.Fatalf("begin build: %v", err) + } + stopKeeping := mc.keepInsertsDuringWipe("/dir") + last, err := mc.deleteFolderChildrenBatch(ctx, "/dir", "") + if err != nil { + t.Fatalf("first batch: %v", err) + } + return func() { + t.Helper() + for last != "" { + if last, err = mc.deleteFolderChildrenBatch(ctx, "/dir", last); err != nil { + t.Fatalf("batch: %v", err) + } + } + stopKeeping() + } +} + +// applyDirEvent applies an event in /dir at tsNs that takes oldName away, puts +// newName there, or both. "" leaves that half out. +func applyDirEvent(t *testing.T, mc *MetaCache, options MetadataResponseApplyOptions, tsNs int64, oldName, newName string) { + t.Helper() + message := &filer_pb.EventNotification{} + if oldName != "" { + message.OldEntry = &filer_pb.Entry{Name: oldName} + } + if newName != "" { + message.NewParentPath = "/dir" + message.NewEntry = &filer_pb.Entry{Name: newName, Attributes: &filer_pb.FuseAttributes{FileMode: 0100644}} + } + event := &filer_pb.SubscribeMetadataResponse{Directory: "/dir", TsNs: tsNs, EventNotification: message} + if err := mc.ApplyMetadataResponse(context.Background(), event, options); err != nil { + t.Fatalf("apply event at %d (%q -> %q): %v", tsNs, oldName, newName, err) + } +} + +// insertListedEntries caches names as children of /dir the way a build's +// listing does. +func insertListedEntries(t *testing.T, mc *MetaCache, names ...string) { + t.Helper() + entries := make([]*filer.Entry, 0, len(names)) + for _, name := range names { + entries = append(entries, &filer.Entry{ + FullPath: util.FullPath("/dir").Child(name), + Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1}, + }) + } + if err := mc.doBatchInsertEntries(context.Background(), entries); err != nil { + t.Fatalf("listing insert: %v", err) + } +} + +// TestRebuildWipeDropsAPlaceholderWhoseFileIsGone: a file created, flushed and +// then unlinked or renamed away, all while the wipe runs, is gone from the +// filer before the listing's snapshot. Neither event is replayed, so the +// removal has to take the placeholder out when it arrives, whichever side of +// the wipe's cursor the name is on. +func TestRebuildWipeDropsAPlaceholderWhoseFileIsGone(t *testing.T) { + for _, tc := range []struct { + desc, placeholder, renamedTo string + }{ + {"unlinked ahead of the cursor", "zz.tmp", ""}, + {"unlinked behind the cursor", "a.tmp", ""}, + {"renamed ahead of the cursor", "zz.tmp", "final"}, + {"renamed behind the cursor", "a.tmp", "final"}, + } { + t.Run(tc.desc, func(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + finishWipe := midRebuildWipe(t, mc) + insertCacheEntry(t, mc, util.FullPath("/dir").Child(tc.placeholder)) + // the flush, then the unlink or the rename + applyDirEvent(t, mc, LocalMetadataResponseApplyOptions, 100, "", tc.placeholder) + applyDirEvent(t, mc, LocalMetadataResponseApplyOptions, 110, tc.placeholder, tc.renamedTo) + finishWipe() + + // The listing starts after the wipe, so its snapshot, 200, is + // past both events. + var want []string + if tc.renamedTo != "" { + insertListedEntries(t, mc, tc.renamedTo) + want = []string{tc.renamedTo} + } + if err := mc.CompleteDirectoryBuild(context.Background(), "/dir", 200, nil); err != nil { + t.Fatalf("complete build: %v", err) + } + if got := cachedChildNames(t, mc, "/dir"); !reflect.DeepEqual(got, want) { + t.Errorf("children of the rebuilt directory = %v, want %v: the filer has no %s", got, want, tc.placeholder) + } + + // The placeholder's version record went with it, so the listing + // fences the name like any other it did not have. + applyDirEvent(t, mc, SubscriberMetadataResponseApplyOptions, 90, "", tc.placeholder) + if got := cachedChildNames(t, mc, "/dir"); !reflect.DeepEqual(got, want) { + t.Errorf("children after an event older than the listing = %v, want %v", got, want) + } + }) + } +} + +// TestRebuildWipeKeepsAPlaceholderRewrittenInPlace: an event that names the +// same path as old and new, a flush or a setattr, vacates nothing. +func TestRebuildWipeKeepsAPlaceholderRewrittenInPlace(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + finishWipe := midRebuildWipe(t, mc) + insertCacheEntry(t, mc, "/dir/zz-open") + applyDirEvent(t, mc, LocalMetadataResponseApplyOptions, 100, "zz-open", "zz-open") + finishWipe() + + if got, want := cachedChildNames(t, mc, "/dir"), []string{"zz-open"}; !reflect.DeepEqual(got, want) { + t.Errorf("children left by the wipe = %v, want %v", got, want) + } +} + +// TestRebuildWipeKeepsAnEntryNewerThanTheRemoval: a removal that arrives during +// the wipe is fenced by versions like any other event, so one older than what +// the wipe kept leaves it alone. +func TestRebuildWipeKeepsAnEntryNewerThanTheRemoval(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + finishWipe := midRebuildWipe(t, mc) + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/zz-newer", + Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 42}, + }, 500); err != nil { + t.Fatalf("insert: %v", err) + } + applyDirEvent(t, mc, SubscriberMetadataResponseApplyOptions, 100, "zz-newer", "") + finishWipe() + + if got, want := cachedChildNames(t, mc, "/dir"), []string{"zz-newer"}; !reflect.DeepEqual(got, want) { + t.Errorf("children left by the wipe = %v, want %v", got, want) + } +} + +// TestRebuildWipeTellsAPlaceholderFromAnEarlierFile: a removal can arrive late, +// after a new file was created under its name. A placeholder is unversioned, so +// only the file the event took away tells whether it is the one cached there. +func TestRebuildWipeTellsAPlaceholderFromAnEarlierFile(t *testing.T) { + for _, tc := range []struct { + desc string + removed *filer_pb.FuseAttributes + renamedTo string + want []string + built []string + }{ + {"another inode", &filer_pb.FuseAttributes{Inode: 7, Crtime: 1}, "", []string{"zz-new"}, []string{"zz-new"}}, + {"another inode, renamed", &filer_pb.FuseAttributes{Inode: 7, Crtime: 1}, "zz-moved", []string{"zz-new"}, []string{"zz-moved", "zz-new"}}, + {"no inode and another creation time", &filer_pb.FuseAttributes{Crtime: 5}, "", []string{"zz-new"}, []string{"zz-new"}}, + {"the same inode", &filer_pb.FuseAttributes{Inode: 42, Crtime: 1}, "", nil, nil}, + {"the same inode, renamed", &filer_pb.FuseAttributes{Inode: 42, Crtime: 1}, "zz-moved", nil, []string{"zz-moved"}}, + {"no inode and the same creation time", &filer_pb.FuseAttributes{Crtime: 1}, "", nil, nil}, + } { + t.Run(tc.desc, func(t *testing.T) { + mc, _, _, invalidations := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + + finishWipe := midRebuildWipe(t, mc) + insertPinnedCacheEntry(t, mc, "/dir/zz-new") // inode 42, created at 1 + removal := &filer_pb.SubscribeMetadataResponse{Directory: "/dir", TsNs: 100, EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "zz-new", Attributes: tc.removed}, + }} + if tc.renamedTo != "" { + removal.EventNotification.NewParentPath = "/dir" + removal.EventNotification.NewEntry = &filer_pb.Entry{Name: tc.renamedTo, Attributes: &filer_pb.FuseAttributes{FileMode: 0100644, Inode: tc.removed.Inode, Crtime: tc.removed.Crtime}} + } + if err := mc.ApplyMetadataResponse(context.Background(), removal, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply removal: %v", err) + } + finishWipe() + + if got := cachedChildNames(t, mc, "/dir"); !reflect.DeepEqual(got, tc.want) { + t.Errorf("children left by the wipe = %v, want %v", got, tc.want) + } + + // An unversioned listing replays every buffered event, so the + // removal the wipe declined must not reach the placeholder there, + // nor the handle that holds it. + mc.WaitForEntryInvalidations() + invalidatedBefore := len(invalidations.paths()) + if err := mc.CompleteDirectoryBuild(context.Background(), "/dir", 0, nil); err != nil { + t.Fatalf("complete build: %v", err) + } + mc.WaitForEntryInvalidations() + if got := cachedChildNames(t, mc, "/dir"); !reflect.DeepEqual(got, tc.built) { + t.Errorf("children after an unversioned listing = %v, want %v", got, tc.built) + } + if tc.want != nil { + for _, path := range invalidations.paths()[invalidatedBefore:] { + if path == "/dir/zz-new" { + t.Errorf("the build invalidated %s for the removal of another file", path) + } + } + } + }) + } +} + +// TestRebuildWipeRechecksADeclinedRemovalOnReplay: a listing attempt that failed +// left file 7 cached and buffered its delete and the create of file 8 at the +// same name. The retry's wipe declines file 8's delete, since file 7 is still +// cached there, but the replay puts file 8 in place first and must then take +// it out again. +func TestRebuildWipeRechecksADeclinedRemovalOnReplay(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + ctx := context.Background() + + fillRebuildWipeDir(t, mc, "/dir", 2*rebuildWipeBatchSize+10) + if err := mc.BeginDirectoryBuild(ctx, "/dir"); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := mc.doBatchInsertEntries(ctx, []*filer.Entry{{ + FullPath: "/dir/zz-same", + Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 7}, + }}); err != nil { + t.Fatalf("first attempt's listing: %v", err) + } + file := func(inode uint64) *filer_pb.Entry { + return &filer_pb.Entry{Name: "zz-same", Attributes: &filer_pb.FuseAttributes{FileMode: 0100644, Inode: inode, Crtime: 1}} + } + apply := func(tsNs int64, message *filer_pb.EventNotification) { + t.Helper() + if err := mc.ApplyMetadataResponse(ctx, &filer_pb.SubscribeMetadataResponse{Directory: "/dir", TsNs: tsNs, EventNotification: message}, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply event at %d: %v", tsNs, err) + } + } + apply(100, &filer_pb.EventNotification{OldEntry: file(7)}) + apply(110, &filer_pb.EventNotification{NewParentPath: "/dir", NewEntry: file(8)}) + + stopKeeping := mc.keepInsertsDuringWipe("/dir") + last, err := mc.deleteFolderChildrenBatch(ctx, "/dir", "") + if err != nil { + t.Fatalf("first batch: %v", err) + } + apply(120, &filer_pb.EventNotification{OldEntry: file(8)}) + for last != "" { + if last, err = mc.deleteFolderChildrenBatch(ctx, "/dir", last); err != nil { + t.Fatalf("batch: %v", err) + } + } + stopKeeping() + + if err := mc.CompleteDirectoryBuild(ctx, "/dir", 0, nil); err != nil { + t.Fatalf("complete build: %v", err) + } + if got := cachedChildNames(t, mc, "/dir"); len(got) != 0 { + t.Errorf("children after an unversioned listing = %v, want none: file 8 was deleted", got) + } +} + +// TestRebuildWipeRenameKeepsTheHardLinkCount: taking a renamed name out of the +// cache removes no link, so the count the other names of the file share stays. +func TestRebuildWipeRenameKeepsTheHardLinkCount(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + mc.SetPinnedChildFn(pinInode42) + ctx := context.Background() + + finishWipe := midRebuildWipe(t, mc) + for _, path := range []util.FullPath{"/dir/zz-link", "/elsewhere/link"} { + if err := mc.InsertEntry(ctx, &filer.Entry{ + FullPath: path, + Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 42}, + HardLinkId: filer.HardLinkId{1, 2, 3, 4}, + HardLinkCounter: 2, + }, 0); err != nil { + t.Fatalf("insert %s: %v", path, err) + } + } + applyDirEvent(t, mc, LocalMetadataResponseApplyOptions, 100, "zz-link", "zz-moved") + finishWipe() + + if got := cachedChildNames(t, mc, "/dir"); len(got) != 0 { + t.Errorf("children left by the wipe = %v, want none: zz-link was renamed", got) + } + other, _, err := mc.FindEntry(ctx, "/elsewhere/link") + if err != nil { + t.Fatalf("find the other link: %v", err) + } + if other.HardLinkCounter != 2 { + t.Errorf("hard link count of the other name = %d, want 2: a rename removes no link", other.HardLinkCounter) + } +} + +// TestRemovalAfterTheWipeIsLeftToTheReplay: once the wipe is over the store +// holds what the listing put there, and a removal from before the snapshot can +// still arrive. If its name was created again by then, the entry it would take +// out is the one the listing just inserted. A removal later than the snapshot +// is replayed. +func TestRemovalAfterTheWipeIsLeftToTheReplay(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + ctx := context.Background() + + if err := mc.BeginDirectoryBuild(ctx, "/dir"); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := mc.deleteFolderChildrenForRebuild(ctx, "/dir"); err != nil { + t.Fatalf("wipe: %v", err) + } + // The listing at 200: "again" was removed at 100 and created again by then. + insertListedEntries(t, mc, "again", "gone-after", "stays") + applyDirEvent(t, mc, SubscriberMetadataResponseApplyOptions, 100, "again", "") + applyDirEvent(t, mc, SubscriberMetadataResponseApplyOptions, 300, "gone-after", "") + if err := mc.CompleteDirectoryBuild(ctx, "/dir", 200, nil); err != nil { + t.Fatalf("complete build: %v", err) + } + + if got, want := cachedChildNames(t, mc, "/dir"), []string{"again", "stays"}; !reflect.DeepEqual(got, want) { + t.Errorf("children of the rebuilt directory = %v, want %v", got, want) + } +} + +// TestRebuildWipeIsRegisteredWhileItRuns: keeping inserts and applying +// removals both act only on a directory whose wipe is registered, so the wipe +// registers itself for as long as it runs, and no longer. +func TestRebuildWipeIsRegisteredWhileItRuns(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true}) + defer mc.Shutdown() + + fillRebuildWipeDir(t, mc, "/dir", rebuildWipeBatchSize+10) + var visited, visitedWhileRegistered int + // called for every child, under the wipe's lock + mc.SetPinnedChildFn(func(*filer.Entry) bool { + visited++ + if _, registered := mc.rebuildWipeInserts["/dir"]; registered { + visitedWhileRegistered++ + } + return false + }) + if err := mc.deleteFolderChildrenForRebuild(context.Background(), "/dir"); err != nil { + t.Fatalf("wipe: %v", err) + } + + if visited == 0 || visitedWhileRegistered != visited { + t.Errorf("the wipe was registered for %d of the %d children it visited, want all of them", visitedWhileRegistered, visited) + } + if len(mc.rebuildWipeInserts) != 0 { + t.Errorf("rebuildWipeInserts = %v after the wipe ended, want empty", mc.rebuildWipeInserts) + } +} diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index bc39caaf86..67d5b10f46 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -157,6 +157,7 @@ type WFS struct { writeBufferAccountant *page_writer.WriteBufferAccountant signature int32 filerCallTimeout time.Duration // bounds one filer call; tests only, 0 = filerRPCTimeout + lookupRebuildWait time.Duration // bounds a lookup's wait on a directory rebuild; tests only, 0 = lookupRebuildTimeout concurrentWriters *util.LimitedConcurrentExecutor copyBufferPool sync.Pool concurrentCopiersSem chan struct{} @@ -633,6 +634,23 @@ func (wfs *WFS) maybeLoadEntry(ctx context.Context, fullpath util.FullPath) (*fi // still allowing recovery once the filer is healthy again. const expiredDirRebuildCooldown = 30 * time.Second +// lookupRebuildTimeout bounds how long a lookup waits on the rebuild it starts. +// The rebuild lists the parent and every expired ancestor up to the mount root, +// and every lookup beneath them joins the same builds, so a slow listing would +// hold a plain stat for as long as it takes. The shared build does not run on +// the requester's context and keeps going after a waiter gives up: this lookup +// falls back to one LookupEntry RPC and later ones find the directory cached. +const lookupRebuildTimeout = 2 * time.Second + +// lookupRebuildWaitTime is lookupRebuildTimeout unless a test set +// lookupRebuildWait on the WFS. +func (wfs *WFS) lookupRebuildWaitTime() time.Duration { + if wfs.lookupRebuildWait > 0 { + return wfs.lookupRebuildWait + } + return lookupRebuildTimeout +} + // lookupEntry looks up an entry by path, checking the local cache first. // Cached metadata is only authoritative when the parent directory itself is cached. // For uncached/read-through directories, always consult the filer directly so stale @@ -658,10 +676,14 @@ func (wfs *WFS) lookupEntry(ctx context.Context, fullpath util.FullPath) (*filer return nil, entryVersion{}, fuse.EINTR } } - if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil { + waitCtx, cancelWait := context.WithTimeout(ctx, wfs.lookupRebuildWaitTime()) + err := wfs.ensureDirectoryVisited(waitCtx, dirPath) + cancelWait() + if err != nil { // Record the attempt so the cooldown suppresses repeated rebuilds - // while the listing keeps failing; once it elapses a later lookup - // retries. Oversized dirs are already marked read-through. + // while the listing keeps failing, and repeated waits while a slow + // one is still running; once it elapses a later lookup retries. + // Oversized dirs are already marked read-through. var tooLarge *meta_cache.DirectoryTooLargeError if !errors.As(err, &tooLarge) { wfs.inodeToPath.MarkRebuildAttempt(dirPath, time.Now()) diff --git a/weed/mount/weedfs_dir_lookup_cache_test.go b/weed/mount/weedfs_dir_lookup_cache_test.go index 296884b73c..0753e324b1 100644 --- a/weed/mount/weedfs_dir_lookup_cache_test.go +++ b/weed/mount/weedfs_dir_lookup_cache_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "runtime" "strconv" "sync" "sync/atomic" @@ -512,3 +513,116 @@ func TestLookupRefreshesStaleSection(t *testing.T) { t.Errorf("ListEntries RPCs = %d, want 1 (one section refresh)", got) } } + +// lookupEntryWithin runs lookupEntry on its own goroutine and reports whether +// it returned before the deadline. The deadline is not a timing assertion: a +// lookup that returns at all returns well inside it, and one that queued on a +// stalled build does not return until the test releases the build. +func lookupEntryWithin(wfs *WFS, fullpath util.FullPath, deadline time.Duration) (name string, status fuse.Status, returned bool) { + type result struct { + name string + status fuse.Status + } + done := make(chan result, 1) + go func() { + entry, _, status := wfs.lookupEntry(context.Background(), fullpath) + r := result{status: status} + if entry != nil { + r.name = entry.Name() + } + done <- r + }() + select { + case r := <-done: + return r.name, r.status, true + case <-time.After(deadline): + return "", fuse.EIO, false + } +} + +// TestLookupDoesNotWaitOutSlowRebuild: a lookup that starts the rebuild of an +// expired directory stops waiting for it after lookupRebuildWaitTime and +// answers with one LookupEntry RPC. The build carries on without it, so once +// the listing completes later lookups are served from the cache again. +func TestLookupDoesNotWaitOutSlowRebuild(t *testing.T) { + wfs := newLookupCacheTestWFS(t, 60) + wfs.lookupRebuildWait = 50 * time.Millisecond + dir := util.FullPath("/dir") + wfs.inodeToPath.Lookup(dir, time.Now().Unix(), true, false, 0, true) + + fake := &lookupCacheTestFiler{ + entries: []*filer_pb.Entry{ + {Name: "a", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}}, + {Name: "b", Attributes: &filer_pb.FuseAttributes{FileSize: 2, FileMode: 0100644}}, + }, + snapshotTsNs: 5000, + } + startFakeFiler(t, wfs, fake) + + if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil { + t.Fatalf("EnsureVisited: %v", err) + } + expireDirCache(t, wfs.inodeToPath, dir) + + // Stall the rebuild's listing until released. + started := make(chan struct{}) + gate := make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(gate) }) } + t.Cleanup(release) + fake.setListGate(gate, started) + + const deadline = 10 * time.Second + lookupCallsBefore := fake.lookupCalls.Load() + name, status, returned := lookupEntryWithin(wfs, dir.Child("a"), deadline) + if !returned { + t.Fatal("lookup is still waiting on a rebuild whose listing is stalled") + } + if status != fuse.OK || name != "a" { + t.Fatalf("lookupEntry a: status=%v name=%q", status, name) + } + if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 1 { + t.Errorf("LookupDirectoryEntry RPCs = %d, want 1 (fall back to the per-entry RPC)", got) + } + // The build runs on its own goroutine and may reach the filer only after + // the lookup stopped waiting for it, so wait for its listing to arrive + // rather than reading how far it got when the lookup returned. From here + // on that listing is in flight, parked at the gate, and already counted. + select { + case <-started: + case <-time.After(deadline): + t.Fatal("lookup should have started a rebuild") + } + + // The wait that timed out counts as a rebuild attempt, so until the + // cooldown passes a lookup goes straight to the per-entry RPC. Were it to + // queue on the build again it could not return: the build is still + // stalled and the wait is now far longer than the deadline. + wfs.lookupRebuildWait = time.Hour + listCallsBefore := fake.listCalls.Load() + if _, status, returned := lookupEntryWithin(wfs, dir.Child("b"), deadline); !returned { + t.Fatal("second lookup queued on the build that is still in flight") + } else if status != fuse.OK { + t.Fatalf("lookupEntry b: %v", status) + } + if got := fake.listCalls.Load() - listCallsBefore; got != 0 { + t.Errorf("second lookup ListEntries RPCs = %d, want 0 (no second rebuild during the cooldown)", got) + } + + // Let the listing finish: the build the first lookup gave up on completes + // and publishes the directory. + release() + for end := time.Now().Add(deadline); !wfs.inodeToPath.IsChildrenCached(dir); { + if time.Now().After(end) { + t.Fatal("the rebuild the lookup stopped waiting for never completed") + } + runtime.Gosched() + } + lookupCallsBefore = fake.lookupCalls.Load() + if name, status, returned := lookupEntryWithin(wfs, dir.Child("a"), deadline); !returned || status != fuse.OK || name != "a" { + t.Fatalf("lookupEntry a after rebuild: returned=%v status=%v name=%q", returned, status, name) + } + if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 0 { + t.Errorf("LookupDirectoryEntry RPCs = %d, want 0 (served from the rebuilt cache)", got) + } +}