diff --git a/weed/command/mount.go b/weed/command/mount.go index 0b2fc929fb..c563e6c757 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -67,6 +67,10 @@ type MountOptions struct { // POSIX compliance options posixDirNlink *bool + // report opens for reading to the filer + accessReport *bool + accessReportInterval *time.Duration + // FUSE performance options writebackCache *bool asyncDio *bool @@ -156,6 +160,9 @@ func init() { // POSIX compliance options mountOptions.posixDirNlink = cmdMount.Flag.Bool("posix.dirNLink", false, "report POSIX-compliant directory nlink (2 + subdirectory count); costs one directory listing per stat") + mountOptions.accessReport = cmdMount.Flag.Bool("accessReport", false, "report opens of a file for reading to the filer with TouchAccessTime, including opens answered from the metadata cache, at most once per -accessReport.interval for each file unless it was modified since its last report. Has no effect unless the filer implements TouchAccessTime") + mountOptions.accessReportInterval = cmdMount.Flag.Duration("accessReport.interval", time.Hour, "with -accessReport, the least time between two reports of the same unmodified file; 0 reports every open") + // FUSE performance options mountOptions.writebackCache = cmdMount.Flag.Bool("writebackCache", false, "enable FUSE writeback cache for improved write performance (at risk of data loss on crash)") mountOptions.asyncDio = cmdMount.Flag.Bool("asyncDio", false, "enable async direct I/O for better concurrency") diff --git a/weed/command/mount_common.go b/weed/command/mount_common.go index a7426c4c25..de30662fc2 100644 --- a/weed/command/mount_common.go +++ b/weed/command/mount_common.go @@ -247,6 +247,8 @@ func buildSeaweedFileSystem(option *MountOptions, p fileSystemParams) *mount.WFS WritebackCache: option.writebackCache != nil && *option.writebackCache, EagerFilerCreate: p.eagerFilerCreate, PosixDirNlink: option.posixDirNlink != nil && *option.posixDirNlink, + AccessReport: option.accessReport != nil && *option.accessReport, + AccessReportInterval: *option.accessReportInterval, // Peer chunk sharing PeerEnabled: option.peerEnabled != nil && *option.peerEnabled, PeerListen: peerStringOrEmpty(option.peerListen), diff --git a/weed/mount/access_report.go b/weed/mount/access_report.go new file mode 100644 index 0000000000..4b592592b5 --- /dev/null +++ b/weed/mount/access_report.go @@ -0,0 +1,274 @@ +package mount + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" + "github.com/seaweedfs/go-fuse/v2/fuse" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +const ( + accessReportQueueSize = 4096 + accessReportWorkers = 8 + // accessReportTimeout bounds one report across all the filers it tries. + accessReportTimeout = 10 * time.Second + // accessReportAttemptTimeout bounds the call to one filer, so a filer that + // accepts the connection but never answers leaves time for the next one. + accessReportAttemptTimeout = 2 * time.Second + // accessReportPause is how long reporting stops after a filer answers + // that it does not implement TouchAccessTime. + accessReportPause = 10 * time.Minute + accessReportLogInterval = time.Minute + // accessReportRecentPaths bounds how many paths remember their last + // report; a path that falls out is reported again on its next open. + accessReportRecentPaths = 1 << 16 +) + +// accessReporter sends a TouchAccessTime for an open of a file for reading +// (-accessReport), at most once per interval for each path unless the file +// was modified since its last report, as relatime does. An open answered +// from the metadata cache never reaches the filer, so without the report a +// filer that tracks access cannot tell a cached file that is still read from +// one that is not. Reports are sent in the background and never hold up the +// open: a report that does not fit in the queue is dropped and counted. +type accessReporter struct { + touch func(ctx context.Context, path util.FullPath, atimeNs int64) error + interval time.Duration + recentMu sync.Mutex + recent *simplelru.LRU[util.FullPath, lastAccessReport] // nil when interval is 0 + queue chan accessReport + ctx context.Context // done once stop is called + cancel context.CancelFunc + pausedUntil atomic.Int64 // unix nanoseconds + dropped atomic.Int64 + failed atomic.Int64 +} + +// accessReport is one open for reading waiting to be sent. +type accessReport struct { + path util.FullPath + atimeNs int64 +} + +// lastAccessReport is when a path was last reported, by this mount's clock, +// and the mtime its entry had then. +type lastAccessReport struct { + reportedNs int64 + mtimeNs int64 +} + +// newAccessReporter returns a reporter that queues reports; start sends them. +// An interval of 0 reports every open. +func newAccessReporter(touch func(ctx context.Context, path util.FullPath, atimeNs int64) error, interval time.Duration) *accessReporter { + ctx, cancel := context.WithCancel(context.Background()) + r := &accessReporter{ + touch: touch, + interval: interval, + queue: make(chan accessReport, accessReportQueueSize), + ctx: ctx, + cancel: cancel, + } + if interval > 0 { + r.recent, _ = simplelru.NewLRU[util.FullPath, lastAccessReport](accessReportRecentPaths, nil) + } + return r +} + +// start starts the workers that send the queued reports. +func (r *accessReporter) start() { + for i := 0; i < accessReportWorkers; i++ { + go r.work() + } + go r.logLosses() +} + +// stop stops the workers and cancels the reports being sent. Reports made +// after stop are dropped. +func (r *accessReporter) stop() { + r.cancel() +} + +// report queues one read of path, last modified at mtimeNs, unless path was +// reported within the interval and not modified since. It does nothing while +// reporting is paused. +func (r *accessReporter) report(path util.FullPath, mtimeNs int64) { + if r.paused() || r.ctx.Err() != nil { + return + } + now := time.Now().UnixNano() + if !r.due(path, now, mtimeNs) { + return + } + select { + case r.queue <- accessReport{path: path, atimeNs: now}: + default: + r.dropped.Add(1) + } +} + +// due reports whether path has no report within the interval or its mtime +// changed since its last report, and if so records the report. Comparing +// mtimes with each other keeps a writer clock that differs from this one from +// making opens due or holding them back. +func (r *accessReporter) due(path util.FullPath, now, mtimeNs int64) bool { + if r.recent == nil { + return true + } + r.recentMu.Lock() + defer r.recentMu.Unlock() + if last, ok := r.recent.Get(path); ok && now-last.reportedNs < int64(r.interval) && mtimeNs == last.mtimeNs { + return false + } + r.recent.Add(path, lastAccessReport{reportedNs: now, mtimeNs: mtimeNs}) + return true +} + +func (r *accessReporter) paused() bool { + return time.Now().UnixNano() < r.pausedUntil.Load() +} + +// work sends queued reports until stop. Reports still queued when reporting +// pauses are dropped: the filer that answered Unimplemented would answer +// them the same way. +func (r *accessReporter) work() { + for { + select { + case <-r.ctx.Done(): + return + case report, ok := <-r.queue: + if !ok { + return + } + if !r.paused() { + r.send(report) + } + } + } +} + +func (r *accessReporter) send(report accessReport) { + ctx, cancel := context.WithTimeout(r.ctx, accessReportTimeout) + defer cancel() + err := r.touch(ctx, report.path, report.atimeNs) + switch { + case err == nil: + case status.Code(err) == codes.Unimplemented: + now := time.Now() + if r.pausedUntil.Swap(now.Add(accessReportPause).UnixNano()) <= now.UnixNano() { + glog.Warningf("access report: filer does not implement TouchAccessTime, pausing reports for %v", accessReportPause) + } + case r.ctx.Err() != nil: + // stopped + default: + r.failed.Add(1) + glog.V(1).Infof("access report %s: %v", report.path, err) + } +} + +func (r *accessReporter) logLosses() { + ticker := time.NewTicker(accessReportLogInterval) + defer ticker.Stop() + for { + select { + case <-r.ctx.Done(): + return + case <-ticker.C: + } + dropped, failed := r.dropped.Swap(0), r.failed.Swap(0) + if dropped > 0 || failed > 0 { + glog.Warningf("access report: %d opens not reported in the last %v (%d dropped on a full queue, %d failed)", + dropped+failed, accessReportLogInterval, dropped, failed) + } + } +} + +// reportOpen reports an open of inode for reading under its current path. +func (wfs *WFS) reportOpen(flags uint32, inode uint64, entry *filer_pb.Entry) { + if wfs.accessReporter == nil { + return + } + if path, status := wfs.inodeToPath.GetPath(inode); status == fuse.OK { + wfs.reportOpenPath(flags, path, entry) + } +} + +// reportOpenPath reports an open of path, whose entry is entry, for reading. +// A write-only open reads nothing and is not reported. +func (wfs *WFS) reportOpenPath(flags uint32, path util.FullPath, entry *filer_pb.Entry) { + if wfs.accessReporter == nil || openFlagsToAccessMask(flags)&fuse.R_OK == 0 || path == "" { + return + } + var mtimeNs int64 + if attr := entry.GetAttributes(); attr != nil { + mtimeNs = attr.Mtime*int64(time.Second) + int64(attr.MtimeNs) + } + wfs.accessReporter.report(path, mtimeNs) +} + +// touchAccessTime sends one report to the filer this mount uses or, if that +// filer cannot be reached in time, to the next one, which the mount then +// keeps using if its selection is still the filer the report started from. +// Any answer, Unimplemented included, ends the walk. +func (wfs *WFS) touchAccessTime(ctx context.Context, path util.FullPath, atimeNs int64) error { + dir, name := path.DirAndName() + req := &filer_pb.TouchAccessTimeRequest{ + Directory: dir, + Name: name, + ClientAtimeNs: atimeNs, + } + maxWait := wfs.filerRPCWait() + if maxWait > accessReportAttemptTimeout { + maxWait = accessReportAttemptTimeout + } + addresses := wfs.option.FilerAddresses + start := int(atomic.LoadInt32(&wfs.option.filerIndex)) + err := fmt.Errorf("no filer to report %s to", path) + for x := 0; x < len(addresses); x++ { + i := (start + x) % len(addresses) + attemptWait := maxWait + if deadline, ok := ctx.Deadline(); ok { + // leave every filer not yet tried an equal share of the report + if share := time.Until(deadline) / time.Duration(len(addresses)-x); share < attemptWait { + attemptWait = share + } + } + err = wfs.callFiler(ctx, false, attemptWait, addresses[i].ToGrpcAddress(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error { + _, err := client.TouchAccessTime(ctx, req) + return err + }) + if err == nil { + atomic.CompareAndSwapInt32(&wfs.option.filerIndex, int32(start), int32(i)) + return nil + } + if !filerUnreachable(err) || ctx.Err() != nil { + return err + } + } + return err +} + +// filerUnreachable reports whether err means the filer did not answer. A +// status the filer sent is its answer, whatever words its message contains. +func filerUnreachable(err error) bool { + st, ok := util.ServerStatus(err) + if !ok { + return util.IsTransientError(err) + } + switch st.Code() { + case codes.Unavailable, codes.ResourceExhausted, codes.DeadlineExceeded: + return true + case codes.Canceled: + return util.IsTransientError(err) + } + return false +} diff --git a/weed/mount/access_report_test.go b/weed/mount/access_report_test.go new file mode 100644 index 0000000000..c8e76ae986 --- /dev/null +++ b/weed/mount/access_report_test.go @@ -0,0 +1,541 @@ +package mount + +import ( + "context" + "slices" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/mount/meta_cache" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// accessReportTestFiler records the reads reported to it. It answers +// TouchAccessTime with err, or, with stall set, not until the caller gives up. +type accessReportTestFiler struct { + *lookupCacheTestFiler + err error + stall bool + onTouch func() + touches chan string +} + +func newAccessReportTestFiler(err error) *accessReportTestFiler { + return &accessReportTestFiler{ + lookupCacheTestFiler: &lookupCacheTestFiler{ + entries: []*filer_pb.Entry{ + {Name: "a", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}}, + {Name: "b", Attributes: &filer_pb.FuseAttributes{FileSize: 2, FileMode: 0100644}}, + {Name: "c", Attributes: &filer_pb.FuseAttributes{FileSize: 3, FileMode: 0100644}}, + }, + snapshotTsNs: 5000, + }, + err: err, + touches: make(chan string, 16), + } +} + +func (s *accessReportTestFiler) TouchAccessTime(ctx context.Context, req *filer_pb.TouchAccessTimeRequest) (*filer_pb.TouchAccessTimeResponse, error) { + s.touches <- string(util.JoinPath(req.Directory, req.Name)) + if s.onTouch != nil { + s.onTouch() + } + if s.stall { + <-ctx.Done() + return nil, ctx.Err() + } + if s.err != nil { + return nil, s.err + } + return &filer_pb.TouchAccessTimeResponse{}, nil +} + +// newAccessReportTestWFS returns a mount on fake with a reporter that is not +// started: its queue holds exactly what was reported until a test sends it. +func newAccessReportTestWFS(t *testing.T, fake *accessReportTestFiler) *WFS { + t.Helper() + wfs := newLookupCacheTestWFS(t, 60) + wfs.openMtimeCache = make(map[uint64][2]int64) + wfs.inodeToPath.Lookup("/dir", time.Now().Unix(), true, false, 0, true) + startFakeFiler(t, wfs, fake) + wfs.accessReporter = newAccessReporter(wfs.touchAccessTime, 0) + t.Cleanup(wfs.accessReporter.stop) + return wfs +} + +// addFiler serves fake as one more filer, after the ones wfs already has. +func addFiler(t *testing.T, wfs *WFS, fake *accessReportTestFiler) { + t.Helper() + existing := wfs.option.FilerAddresses + startFakeFiler(t, wfs, fake) + wfs.option.FilerAddresses = append(append([]pb.ServerAddress(nil), existing...), wfs.option.FilerAddresses...) +} + +// queuedReports takes the reports waiting in r's queue. +func queuedReports(r *accessReporter) []string { + var paths []string + for { + select { + case report := <-r.queue: + paths = append(paths, string(report.path)) + default: + return paths + } + } +} + +func openForTest(t *testing.T, wfs *WFS, name string, flags uint32) { + t.Helper() + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir").Child(name), time.Now().Unix(), false, false, 0, true) + out := &fuse.OpenOut{} + if status := wfs.Open(nil, &fuse.OpenIn{InHeader: fuse.InHeader{NodeId: inode}, Flags: flags}, out); status != fuse.OK { + t.Fatalf("open %s: %v", name, status) + } + wfs.fhMap.ReleaseByHandle(FileHandleId(out.Fh)) +} + +// TestEachOpenForReadingIsReportedOnce checks that every open of a file for +// reading is reported once, also when the open is answered from the metadata +// cache and the filer sees no lookup, and that a write-only open is not. +func TestEachOpenForReadingIsReportedOnce(t *testing.T) { + fake := newAccessReportTestFiler(nil) + wfs := newAccessReportTestWFS(t, fake) + + openForTest(t, wfs, "a", syscall.O_RDONLY) + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/a"}; !slices.Equal(got, want) { + t.Fatalf("reports = %v, want %v", got, want) + } + + if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, "/dir", 0); err != nil { + t.Fatalf("EnsureVisited: %v", err) + } + openForTest(t, wfs, "b", syscall.O_RDONLY) + openForTest(t, wfs, "b", syscall.O_RDONLY) + openForTest(t, wfs, "a", syscall.O_RDWR) + openForTest(t, wfs, "c", syscall.O_WRONLY) + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/b", "/dir/b", "/dir/a"}; !slices.Equal(got, want) { + t.Fatalf("reports = %v, want %v", got, want) + } + if lookups := fake.lookupCalls.Load(); lookups != 1 { + t.Fatalf("lookups = %d, want 1: opens in a cached directory are answered locally", lookups) + } +} + +// TestOpensWithinTheIntervalAreReportedOnce checks that a path is reported +// on its first open and again only once the interval has passed. +func TestOpensWithinTheIntervalAreReportedOnce(t *testing.T) { + wfs := newAccessReportTestWFS(t, newAccessReportTestFiler(nil)) + wfs.accessReporter = newAccessReporter(wfs.touchAccessTime, time.Hour) + t.Cleanup(wfs.accessReporter.stop) + + openForTest(t, wfs, "a", syscall.O_RDONLY) + openForTest(t, wfs, "a", syscall.O_RDONLY) + openForTest(t, wfs, "b", syscall.O_RDWR) + openForTest(t, wfs, "a", syscall.O_RDONLY) + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/a", "/dir/b"}; !slices.Equal(got, want) { + t.Fatalf("reports = %v, want %v", got, want) + } + + last, _ := wfs.accessReporter.recent.Get("/dir/a") + last.reportedNs -= int64(time.Hour) + wfs.accessReporter.recent.Add("/dir/a", last) + openForTest(t, wfs, "a", syscall.O_RDONLY) + openForTest(t, wfs, "b", syscall.O_RDONLY) + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/a"}; !slices.Equal(got, want) { + t.Fatalf("reports after the interval = %v, want %v", got, want) + } +} + +// TestOpenAfterAWriteIsReportedWithinTheInterval checks that a file modified +// since its last report is reported again on its next open within the +// interval, as relatime updates atime on the first read after a write, and +// that a writer clock ahead of the mount's does not make every open due. +func TestOpenAfterAWriteIsReportedWithinTheInterval(t *testing.T) { + wfs := newAccessReportTestWFS(t, newAccessReportTestFiler(nil)) + wfs.accessReporter = newAccessReporter(wfs.touchAccessTime, time.Hour) + t.Cleanup(wfs.accessReporter.stop) + if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, "/dir", 0); err != nil { + t.Fatalf("EnsureVisited: %v", err) + } + modify := func(mtime time.Time, versionTsNs int64) { + entry := &filer_pb.Entry{Name: "a", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644, + Mtime: mtime.Unix(), MtimeNs: int32(mtime.Nanosecond())}} + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", entry), versionTsNs); err != nil { + t.Fatalf("InsertEntry: %v", err) + } + } + + openForTest(t, wfs, "a", syscall.O_RDONLY) + openForTest(t, wfs, "a", syscall.O_RDONLY) + modify(time.Now().Add(time.Millisecond), 6000) + openForTest(t, wfs, "a", syscall.O_RDONLY) + openForTest(t, wfs, "a", syscall.O_RDONLY) + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/a", "/dir/a"}; !slices.Equal(got, want) { + t.Fatalf("reports = %v, want %v", got, want) + } + + modify(time.Now().Add(time.Minute), 7000) + openForTest(t, wfs, "a", syscall.O_RDONLY) + openForTest(t, wfs, "a", syscall.O_RDONLY) + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/a"}; !slices.Equal(got, want) { + t.Fatalf("reports with the writer clock ahead = %v, want %v", got, want) + } +} + +// TestMtimeAheadDoesNotExtendTheInterval checks that a file dated in the +// future is reported again once the interval has passed by this mount's +// clock. +func TestMtimeAheadDoesNotExtendTheInterval(t *testing.T) { + r := newAccessReporter(nil, time.Hour) + t.Cleanup(r.stop) + start := time.Now().UnixNano() + mtime := start + int64(24*time.Hour) + for _, step := range []struct { + after time.Duration + due bool + }{{0, true}, {time.Minute, false}, {time.Hour, true}, {time.Hour + time.Minute, false}} { + if got := r.due("/dir/a", start+int64(step.after), mtime); got != step.due { + t.Fatalf("due after %v = %v, want %v", step.after, got, step.due) + } + } +} + +// TestCreateOfAnExistingFileIsReported checks the opens that reach Create, +// such as O_CREAT|O_RDWR of a file that already exists: they are reported +// like opens, and only with read access. +func TestCreateOfAnExistingFileIsReported(t *testing.T) { + wfs := newAccessReportTestWFS(t, newAccessReportTestFiler(nil)) + dir := wfs.inodeToPath.Lookup("/dir", time.Now().Unix(), true, false, 0, true) + + for _, flags := range []uint32{syscall.O_RDWR | syscall.O_CREAT, syscall.O_WRONLY | syscall.O_CREAT} { + out := &fuse.CreateOut{} + if code := wfs.Create(nil, &fuse.CreateIn{InHeader: fuse.InHeader{NodeId: dir}, Flags: flags}, "a", out); code != fuse.OK { + t.Fatalf("create with flags %o: %v", flags, code) + } + wfs.fhMap.ReleaseByHandle(FileHandleId(out.Fh)) + } + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/a"}; !slices.Equal(got, want) { + t.Fatalf("reports = %v, want %v", got, want) + } +} + +// TestCreateThroughAHardLinkReportsItsName checks that a create opening an +// existing hard link reports the name it was given, not the inode's first +// known name. +func TestCreateThroughAHardLinkReportsItsName(t *testing.T) { + fake := newAccessReportTestFiler(nil) + for _, name := range []string{"h1", "h2"} { + fake.entries = append(fake.entries, &filer_pb.Entry{Name: name, HardLinkId: []byte("link"), + Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644, Inode: 4242}}) + } + wfs := newAccessReportTestWFS(t, fake) + dir := wfs.inodeToPath.Lookup("/dir", time.Now().Unix(), true, false, 0, true) + wfs.inodeToPath.Lookup("/dir/h1", time.Now().Unix(), false, true, 4242, true) + if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, "/dir", 0); err != nil { + t.Fatalf("EnsureVisited: %v", err) + } + + out := &fuse.CreateOut{} + if code := wfs.Create(nil, &fuse.CreateIn{InHeader: fuse.InHeader{NodeId: dir}, Flags: syscall.O_RDWR | syscall.O_CREAT}, "h2", out); code != fuse.OK { + t.Fatalf("create: %v", code) + } + wfs.fhMap.ReleaseByHandle(FileHandleId(out.Fh)) + if out.NodeId != 4242 { + t.Fatalf("create opened inode %d, want the hard link's 4242", out.NodeId) + } + if got, want := queuedReports(wfs.accessReporter), []string{"/dir/h2"}; !slices.Equal(got, want) { + t.Fatalf("reports = %v, want %v", got, want) + } +} + +func TestNoReportsWithoutAccessReport(t *testing.T) { + fake := newAccessReportTestFiler(nil) + wfs := newAccessReportTestWFS(t, fake) + wfs.accessReporter = nil + + openForTest(t, wfs, "a", syscall.O_RDONLY) + if n := len(fake.touches); n != 0 { + t.Fatalf("filer got %d reports, want none", n) + } +} + +func TestReportsReachTheFiler(t *testing.T) { + fake := newAccessReportTestFiler(nil) + wfs := newAccessReportTestWFS(t, fake) + wfs.accessReporter.start() + + openForTest(t, wfs, "a", syscall.O_RDONLY) + select { + case path := <-fake.touches: + if path != "/dir/a" { + t.Fatalf("report for %s, want /dir/a", path) + } + case <-time.After(5 * time.Second): + t.Fatal("no report reached the filer") + } +} + +// TestReportingPausesWhenTheFilerHasNoHandler checks that a filer answering +// Unimplemented is not sent a report for every later open. +func TestReportingPausesWhenTheFilerHasNoHandler(t *testing.T) { + fake := newAccessReportTestFiler(status.Error(codes.Unimplemented, "method TouchAccessTime not implemented")) + wfs := newAccessReportTestWFS(t, fake) + r := wfs.accessReporter + + openForTest(t, wfs, "a", syscall.O_RDONLY) + select { + case report := <-r.queue: + r.send(report) + default: + t.Fatal("the open was not reported") + } + if !r.paused() { + t.Fatal("reporting did not pause after Unimplemented") + } + openForTest(t, wfs, "b", syscall.O_RDONLY) + if got := queuedReports(r); len(got) != 0 { + t.Fatalf("reported %v while paused", got) + } + if n := len(fake.touches); n != 1 { + t.Fatalf("filer got %d reports, want 1", n) + } +} + +// TestQueuedReportsAreDroppedWhilePaused checks that reports queued before the +// pause are not sent to the filer that answered Unimplemented. +func TestQueuedReportsAreDroppedWhilePaused(t *testing.T) { + var sent atomic.Int32 + r := newAccessReporter(func(ctx context.Context, path util.FullPath, atimeNs int64) error { + sent.Add(1) + return nil + }, 0) + defer r.stop() + r.report("/dir/a", 0) + r.report("/dir/b", 0) + r.pausedUntil.Store(time.Now().Add(time.Hour).UnixNano()) + + close(r.queue) // work returns once the queue is drained + r.work() + if n := sent.Load(); n != 0 { + t.Fatalf("sent %d queued reports during the pause, want 0", n) + } +} + +// TestReportDoesNotBlockOnAFullQueue checks that report drops and counts what +// does not fit instead of waiting for room. +func TestReportDoesNotBlockOnAFullQueue(t *testing.T) { + r := newAccessReporter(func(ctx context.Context, path util.FullPath, atimeNs int64) error { return nil }, 0) + defer r.stop() + + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < accessReportQueueSize+1; i++ { + r.report("/dir/a", 0) + } + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("report blocked on a full queue") + } + if dropped := r.dropped.Load(); dropped != 1 { + t.Fatalf("dropped = %d, want 1", dropped) + } +} + +func TestStopCancelsTheReportInFlight(t *testing.T) { + entered, cancelled := make(chan struct{}), make(chan struct{}) + r := newAccessReporter(func(ctx context.Context, path util.FullPath, atimeNs int64) error { + close(entered) + <-ctx.Done() + close(cancelled) + return ctx.Err() + }, 0) + r.start() + r.report("/dir/a", 0) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("the report was not sent") + } + + r.stop() + select { + case <-cancelled: + case <-time.After(5 * time.Second): + t.Fatal("stop did not cancel the report in flight") + } +} + +// TestReportAfterStopIsDropped uses a reporter without workers, so a report +// that stop failed to refuse stays in the queue. +func TestReportAfterStopIsDropped(t *testing.T) { + r := newAccessReporter(func(ctx context.Context, path util.FullPath, atimeNs int64) error { return nil }, 0) + r.stop() + r.report("/dir/b", 0) + if got := queuedReports(r); len(got) != 0 { + t.Fatalf("queued %v after stop", got) + } +} + +// TestTouchAccessTimeDoesNotFailOverOnAnAnswer checks that a filer that +// answers, even with an error, is not followed by the next filer. +func TestTouchAccessTimeDoesNotFailOverOnAnAnswer(t *testing.T) { + for _, answer := range []struct { + code codes.Code + message string + }{ + {codes.Unimplemented, "refused"}, + {codes.Unimplemented, "rate limit reporting not implemented"}, + {codes.InvalidArgument, "refused"}, + {codes.InvalidArgument, "invalid path /transport"}, + } { + code := answer.code + t.Run(code.String()+" "+answer.message, func(t *testing.T) { + first := newAccessReportTestFiler(status.Error(code, answer.message)) + wfs := newAccessReportTestWFS(t, first) + second := newAccessReportTestFiler(nil) + addFiler(t, wfs, second) + + ctx, cancel := context.WithTimeout(context.Background(), accessReportTimeout) + defer cancel() + if err := wfs.touchAccessTime(ctx, "/dir/a", time.Now().UnixNano()); status.Code(err) != code { + t.Fatalf("err = %v, want %v", err, code) + } + if len(first.touches) != 1 || len(second.touches) != 0 { + t.Fatalf("reports: first filer %d, second %d, want 1 and 0", len(first.touches), len(second.touches)) + } + }) + } +} + +// TestRejectedReportKeepsTheFilerChannel checks that a filer rejecting a +// report with a message that reads like a transport failure does not close +// the channel other calls to that filer are still using. +func TestRejectedReportKeepsTheFilerChannel(t *testing.T) { + fake := newAccessReportTestFiler(status.Error(codes.InvalidArgument, "invalid path /transport")) + inFlight, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + fake.onTouch = func() { + if calls.Add(1) == 1 { + close(inFlight) + <-release + } + } + wfs := newAccessReportTestWFS(t, fake) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + + ctx, cancel := context.WithTimeout(context.Background(), accessReportTimeout) + defer cancel() + // The call held in flight gets the whole test deadline, not a report + // attempt's share, so a slow runner cannot time it out on a healthy channel. + first := make(chan error, 1) + go func() { + first <- wfs.callFiler(ctx, false, accessReportTimeout, wfs.option.FilerAddresses[0].ToGrpcAddress(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error { + _, err := client.TouchAccessTime(ctx, &filer_pb.TouchAccessTimeRequest{Directory: "/dir", Name: "a", ClientAtimeNs: time.Now().UnixNano()}) + return err + }) + }() + select { + case <-inFlight: + case err := <-first: + t.Fatalf("first call returned %v before reaching the filer", err) + case <-ctx.Done(): + t.Fatal("first call never reached the filer") + } + if err := wfs.touchAccessTime(ctx, "/dir/transport", time.Now().UnixNano()); status.Code(err) != codes.InvalidArgument { + t.Fatalf("rejected report: err = %v, want InvalidArgument", err) + } + unblock() + if err := <-first; status.Code(err) != codes.InvalidArgument { + t.Fatalf("call in flight on the same channel: err = %v, want the filer's InvalidArgument", err) + } +} + +// TestTouchAccessTimeFailsOverPastAStalledFiler checks that a filer that never +// answers uses up one attempt, not the whole report, even though one filer +// call may otherwise take longer than a report, and that the mount then keeps +// the filer that answered. +func TestTouchAccessTimeFailsOverPastAStalledFiler(t *testing.T) { + stalled := newAccessReportTestFiler(nil) + stalled.stall = true + wfs := newAccessReportTestWFS(t, stalled) + if wfs.filerRPCWait() < accessReportTimeout { + t.Fatalf("filer call bound %v is below the report bound %v; the test would not stall", wfs.filerRPCWait(), accessReportTimeout) + } + healthy := newAccessReportTestFiler(nil) + addFiler(t, wfs, healthy) + + ctx, cancel := context.WithTimeout(context.Background(), accessReportTimeout) + defer cancel() + if err := wfs.touchAccessTime(ctx, "/dir/a", time.Now().UnixNano()); err != nil { + t.Fatalf("report did not reach the healthy filer: %v", err) + } + if len(stalled.touches) != 1 || len(healthy.touches) != 1 { + t.Fatalf("reports: stalled filer %d, healthy filer %d, want 1 and 1", len(stalled.touches), len(healthy.touches)) + } + if i := atomic.LoadInt32(&wfs.option.filerIndex); i != 1 { + t.Fatalf("filerIndex = %d, want 1, the filer that answered", i) + } +} + +// TestTouchAccessTimeKeepsANewerFilerSelection checks that a report that +// failed over does not move the mount back from a filer another call +// selected while the report was in flight. +func TestTouchAccessTimeKeepsANewerFilerSelection(t *testing.T) { + down := newAccessReportTestFiler(status.Error(codes.Unavailable, "down")) + wfs := newAccessReportTestWFS(t, down) + answering := newAccessReportTestFiler(nil) + addFiler(t, wfs, answering) + addFiler(t, wfs, newAccessReportTestFiler(nil)) + answering.onTouch = func() { atomic.StoreInt32(&wfs.option.filerIndex, 2) } + + ctx, cancel := context.WithTimeout(context.Background(), accessReportTimeout) + defer cancel() + if err := wfs.touchAccessTime(ctx, "/dir/a", time.Now().UnixNano()); err != nil { + t.Fatalf("report: %v", err) + } + if len(down.touches) != 1 || len(answering.touches) != 1 { + t.Fatalf("reports: unavailable filer %d, answering filer %d, want 1 and 1", len(down.touches), len(answering.touches)) + } + if i := atomic.LoadInt32(&wfs.option.filerIndex); i != 2 { + t.Fatalf("filerIndex = %d, want 2, the filer selected during the report", i) + } +} + +// TestTouchAccessTimeLeavesTimeForTheLastFiler checks that stalled filers +// share the report's time with the filers after them instead of each taking +// a full attempt, so a healthy last filer is still reached. +func TestTouchAccessTimeLeavesTimeForTheLastFiler(t *testing.T) { + first := newAccessReportTestFiler(nil) + first.stall = true + wfs := newAccessReportTestWFS(t, first) + second := newAccessReportTestFiler(nil) + second.stall = true + addFiler(t, wfs, second) + healthy := newAccessReportTestFiler(nil) + addFiler(t, wfs, healthy) + + reportTime := 3 * accessReportAttemptTimeout / 4 + ctx, cancel := context.WithTimeout(context.Background(), reportTime) + defer cancel() + if err := wfs.touchAccessTime(ctx, "/dir/a", time.Now().UnixNano()); err != nil { + t.Fatalf("report did not reach the last filer within %v: %v", reportTime, err) + } + if len(first.touches) != 1 || len(second.touches) != 1 || len(healthy.touches) != 1 { + t.Fatalf("reports: %d, %d, %d, want one to each filer", len(first.touches), len(second.touches), len(healthy.touches)) + } +} diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index bb9ecc63bf..b50907731c 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -128,6 +128,15 @@ type Option struct { // When false (default), directories report nlink=2. PosixDirNlink bool + // AccessReport reports opens of a file for reading to the filer with + // TouchAccessTime, at most once per AccessReportInterval for each path + // (0 reports every open), so a filer that tracks access also sees the + // reads answered from the metadata cache. It has no effect unless the + // filer implements TouchAccessTime: while the filer answers + // Unimplemented, reporting pauses. + AccessReport bool + AccessReportInterval time.Duration + uniqueCacheDirForRead string uniqueCacheDirForWrite string } @@ -156,6 +165,7 @@ type WFS struct { dhMap *DirectoryHandleToInode fuseServer *fuse.Server IsOverQuota bool + accessReporter *accessReporter // nil unless -accessReport // Handle lock nesting order is defined at FileHandle.entryLock. fhLockTable *util.LockTable[FileHandleId] hardLinkLockTable *util.LockTable[string] @@ -299,6 +309,11 @@ func NewSeaweedFileSystem(option *Option) *WFS { glog.V(0).Infof("distributed lock manager disabled: writeback cache implies single-writer mode") } + if option.AccessReport { + wfs.accessReporter = newAccessReporter(wfs.touchAccessTime, option.AccessReportInterval) + wfs.accessReporter.start() + } + wfs.option.filerIndex = int32(rand.IntN(len(option.FilerAddresses))) wfs.option.setupUniqueCacheDirectory() if option.CacheSizeMBForRead > 0 { @@ -346,6 +361,9 @@ func NewSeaweedFileSystem(option *Option) *WFS { os.RemoveAll(option.getUniqueCacheDirForWrite()) } os.RemoveAll(option.getUniqueCacheDirForRead()) + if wfs.accessReporter != nil { + wfs.accessReporter.stop() + } if wfs.peerAnnouncer != nil { wfs.peerAnnouncer.Stop() } diff --git a/weed/mount/weedfs_file_io.go b/weed/mount/weedfs_file_io.go index 019e80d0c0..2442000ee9 100644 --- a/weed/mount/weedfs_file_io.go +++ b/weed/mount/weedfs_file_io.go @@ -72,6 +72,7 @@ func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) if status == fuse.OK { out.Fh = uint64(openFh) out.OpenFlags = 0 + wfs.reportOpen(in.Flags, in.NodeId, fileHandle.GetEntry().GetEntry()) // For read-only opens, set FOPEN_KEEP_CACHE when the file's mtime // has not changed since the last open. This tells the kernel to diff --git a/weed/mount/weedfs_file_mkrm.go b/weed/mount/weedfs_file_mkrm.go index 11e0d3831c..dc9a91b1ab 100644 --- a/weed/mount/weedfs_file_mkrm.go +++ b/weed/mount/weedfs_file_mkrm.go @@ -172,6 +172,7 @@ func (wfs *WFS) openExistingForCreate(ctx context.Context, in *fuse.CreateIn, en wfs.outputPbEntry(&out.EntryOut, inode, entry) out.Fh = uint64(openFh) out.OpenFlags = 0 + wfs.reportOpenPath(in.Flags, entryFullPath, entry) return fuse.OK } diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 6fb1c79b54..156ecdf0a2 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -496,6 +496,12 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool { // in-flight RPC with "the client connection is closing"; keep it and let // the caller retry. return false + case codes.InvalidArgument, codes.NotFound, codes.AlreadyExists, + codes.FailedPrecondition, codes.OutOfRange, codes.DataLoss: + // gRPC never raises these itself, so the peer answered. Its + // message can name a path such as /transport, which must not + // reach the string matching below. + return false case codes.Canceled, codes.DeadlineExceeded: // Ambiguous: a stale cached channel rejects RPCs this way (a peer // restart behind a k8s Service VIP), and so does the RPC's own context diff --git a/weed/pb/grpc_client_server_test.go b/weed/pb/grpc_client_server_test.go index a544392c86..8bcf4e6cf3 100644 --- a/weed/pb/grpc_client_server_test.go +++ b/weed/pb/grpc_client_server_test.go @@ -114,6 +114,19 @@ func TestShouldInvalidateConnection_ResourceExhaustedIsPerRequest(t *testing.T) } } +func TestShouldInvalidateConnection_PeerAnswerNamingTransportKeepsChannel(t *testing.T) { + for _, code := range []codes.Code{codes.InvalidArgument, codes.NotFound, codes.AlreadyExists, + codes.FailedPrecondition, codes.OutOfRange, codes.DataLoss} { + answer := status.Error(code, "invalid path /transport/dns: connection refused") + if shouldInvalidateConnection(context.Background(), answer) { + t.Errorf("%v answer from the peer must not invalidate the shared connection", code) + } + if shouldInvalidateConnection(context.Background(), fmt.Errorf("touch: %w", answer)) { + t.Errorf("wrapped %v answer from the peer must not invalidate the shared connection", code) + } + } +} + // TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the // marshal-error carve-out does not swallow real server-side Internal errors, // which previously caused — and should continue to cause — connection