From 72ac9e96b2f74da6c0766e1ed2a77289ba1c49d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=A9=AC=E4=B8=80=E5=8F=AA?= Date: Thu, 19 Nov 2026 02:58:05 +0800 Subject: [PATCH] mount: answer an interrupted read with EINTR, not EIO (#12723) * mount: answer an interrupted read with EINTR, not EIO The kernel's FUSE contract (Documentation/filesystems/fuse/fuse.rst, "Interrupting filesystem operations") lets a filesystem either ignore an INTERRUPT or honor it by replying EINTR to the original request. Read did neither: the FUSE cancel channel cancels the read's context, the chunk fetch fails with "context canceled", and every error became EIO. Interrupts come from any signal the reader handles. Go's runtime sends SIGURG for goroutine preemption, so Go programs reading under load hit this on healthy data: on a 128-thread 64 KiB random-read test about 250 of 5 million reads failed with EIO, and none with GODEBUG=asyncpreemptoff=1. Read now returns EINTR when its own context was cancelled, the way dirReadStatus already does for directory reads and lookups do since #11717. Go's os.File retries EINTR, so those readers see no error. A cancelled read also skips the entry refresh retry, and neither read log line reports an interrupt as a warning or error. Failures without an interrupt stay EIO. Tests: TestReadInterruptedReturnsEINTR (fails on the old code with EIO), TestReadFailureWithoutInterruptReturnsEIO; weed/mount/... passes. Co-Authored-By: Claude Opus 5.5 * mount: require the failed-read test to reach the filer Co-Authored-By: Claude Opus 5.5 * mount: skip the read retry once the request is interrupted Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- weed/mount/filehandle_read.go | 2 +- weed/mount/weedfs_file_read.go | 12 ++- weed/mount/weedfs_file_read_cancel_test.go | 109 +++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 weed/mount/weedfs_file_read_cancel_test.go diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go index 47e785fe5b..35d3339be6 100644 --- a/weed/mount/filehandle_read.go +++ b/weed/mount/filehandle_read.go @@ -103,7 +103,7 @@ func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte fh.refreshChunkView() totalRead, ts, err := fh.entryChunkGroup.ReadDataAt(ctx, fileSize, buff, offset) - if err != nil && err != io.EOF { + if err != nil && err != io.EOF && ctx.Err() == nil { glog.Errorf("file handle read %s: %v", fileFullPath, err) } diff --git a/weed/mount/weedfs_file_read.go b/weed/mount/weedfs_file_read.go index c5cf9aa5ad..b854c987de 100644 --- a/weed/mount/weedfs_file_read.go +++ b/weed/mount/weedfs_file_read.go @@ -52,7 +52,7 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse offset := int64(in.Offset) totalRead, err := readDataByFileHandleWithContext(ctx, buff, fh, offset) - if err != nil { + if err != nil && ctx.Err() == nil { // The chunk views a read resolves can name data a concurrent overwrite // already replaced: the filer queues replaced file ids for deletion at // commit while this handle only learns the new list when the metadata @@ -61,11 +61,19 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) retry := fh.refreshEntryForRead(ctx) fhActiveLock = fh.wfs.fhLockTable.AcquireLock("Read", fh.fh, util.SharedLock) - if retry { + if retry && ctx.Err() == nil { totalRead, err = readDataByFileHandleWithContext(ctx, buff, fh, offset) } } if err != nil { + // The kernel's FUSE contract answers an interrupted request with EINTR + // on the original request, never EIO. Interrupts come from any signal + // the reader handles, Go's preemption SIGURG included, so they are + // routine and not worth a warning. + if ctx.Err() != nil { + glog.V(3).Infof("file handle read %s %d: interrupted: %v", fh.FullPath(), totalRead, err) + return nil, fuse.EINTR + } glog.Warningf("file handle read %s %d: %v", fh.FullPath(), totalRead, err) return nil, fuse.EIO } diff --git a/weed/mount/weedfs_file_read_cancel_test.go b/weed/mount/weedfs_file_read_cancel_test.go new file mode 100644 index 0000000000..1564c37227 --- /dev/null +++ b/weed/mount/weedfs_file_read_cancel_test.go @@ -0,0 +1,109 @@ +package mount + +import ( + "context" + "testing" + "time" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// remoteReadFiler answers the download of an uncached remote file: stalled +// parks it until the caller's context ends, otherwise it fails at once. +type remoteReadFiler struct { + filer_pb.UnimplementedSeaweedFilerServer + stalled bool + started chan struct{} +} + +func (s *remoteReadFiler) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) { + select { + case <-s.started: + default: + close(s.started) + } + if !s.stalled { + return nil, status.Error(codes.NotFound, "remote object gone") + } + <-ctx.Done() + return nil, status.Error(codes.Canceled, ctx.Err().Error()) +} + +// startRemoteRead opens /dir/file as an uncached remote entry and reads it +// in the background with the given FUSE cancel channel. +func startRemoteRead(t *testing.T, fake *remoteReadFiler, cancel chan struct{}) <-chan fuse.Status { + t.Helper() + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, fake) + + root := util.FullPath("/") + wfs.inodeToPath.MarkChildrenCached(root) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 4096}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 4096}, + }, 0, 0) + + done := make(chan fuse.Status, 1) + go func() { + _, code := wfs.Read(cancel, &fuse.ReadIn{ + InHeader: fuse.InHeader{NodeId: inode}, + Fh: uint64(fh.fh), + Size: 4096, + }, make([]byte, 4096)) + done <- code + }() + return done +} + +// The kernel's FUSE contract answers an interrupted request with EINTR on the +// original request. Interrupts come from any handled signal, Go's preemption +// SIGURG included, so EIO here would fail healthy reads. +func TestReadInterruptedReturnsEINTR(t *testing.T) { + fake := &remoteReadFiler{stalled: true, started: make(chan struct{})} + cancel := make(chan struct{}) + done := startRemoteRead(t, fake, cancel) + + select { + case <-fake.started: + case <-time.After(10 * time.Second): + t.Fatal("read never reached the filer") + } + close(cancel) + + select { + case code := <-done: + if code != fuse.EINTR { + t.Fatalf("interrupted read returned %v, want EINTR", code) + } + case <-time.After(10 * time.Second): + t.Fatal("Read still blocked after the FUSE request was interrupted") + } +} + +// A read that fails without an interrupt is a real I/O error and stays EIO. +func TestReadFailureWithoutInterruptReturnsEIO(t *testing.T) { + fake := &remoteReadFiler{started: make(chan struct{})} + done := startRemoteRead(t, fake, make(chan struct{})) + + select { + case code := <-done: + if code != fuse.EIO { + t.Fatalf("failed read returned %v, want EIO", code) + } + select { + case <-fake.started: + default: + t.Fatal("read failed before it reached the filer") + } + case <-time.After(30 * time.Second): + t.Fatal("failed read never returned") + } +}