diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go index 47e785fe5b..35d3339be6 100644 --- a/weed/mount/filehandle_read.go +++ b/weed/mount/filehandle_read.go @@ -103,7 +103,7 @@ func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte fh.refreshChunkView() totalRead, ts, err := fh.entryChunkGroup.ReadDataAt(ctx, fileSize, buff, offset) - if err != nil && err != io.EOF { + if err != nil && err != io.EOF && ctx.Err() == nil { glog.Errorf("file handle read %s: %v", fileFullPath, err) } diff --git a/weed/mount/weedfs_file_read.go b/weed/mount/weedfs_file_read.go index c5cf9aa5ad..b854c987de 100644 --- a/weed/mount/weedfs_file_read.go +++ b/weed/mount/weedfs_file_read.go @@ -52,7 +52,7 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse offset := int64(in.Offset) totalRead, err := readDataByFileHandleWithContext(ctx, buff, fh, offset) - if err != nil { + if err != nil && ctx.Err() == nil { // The chunk views a read resolves can name data a concurrent overwrite // already replaced: the filer queues replaced file ids for deletion at // commit while this handle only learns the new list when the metadata @@ -61,11 +61,19 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) retry := fh.refreshEntryForRead(ctx) fhActiveLock = fh.wfs.fhLockTable.AcquireLock("Read", fh.fh, util.SharedLock) - if retry { + if retry && ctx.Err() == nil { totalRead, err = readDataByFileHandleWithContext(ctx, buff, fh, offset) } } if err != nil { + // The kernel's FUSE contract answers an interrupted request with EINTR + // on the original request, never EIO. Interrupts come from any signal + // the reader handles, Go's preemption SIGURG included, so they are + // routine and not worth a warning. + if ctx.Err() != nil { + glog.V(3).Infof("file handle read %s %d: interrupted: %v", fh.FullPath(), totalRead, err) + return nil, fuse.EINTR + } glog.Warningf("file handle read %s %d: %v", fh.FullPath(), totalRead, err) return nil, fuse.EIO } diff --git a/weed/mount/weedfs_file_read_cancel_test.go b/weed/mount/weedfs_file_read_cancel_test.go new file mode 100644 index 0000000000..1564c37227 --- /dev/null +++ b/weed/mount/weedfs_file_read_cancel_test.go @@ -0,0 +1,109 @@ +package mount + +import ( + "context" + "testing" + "time" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// remoteReadFiler answers the download of an uncached remote file: stalled +// parks it until the caller's context ends, otherwise it fails at once. +type remoteReadFiler struct { + filer_pb.UnimplementedSeaweedFilerServer + stalled bool + started chan struct{} +} + +func (s *remoteReadFiler) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) { + select { + case <-s.started: + default: + close(s.started) + } + if !s.stalled { + return nil, status.Error(codes.NotFound, "remote object gone") + } + <-ctx.Done() + return nil, status.Error(codes.Canceled, ctx.Err().Error()) +} + +// startRemoteRead opens /dir/file as an uncached remote entry and reads it +// in the background with the given FUSE cancel channel. +func startRemoteRead(t *testing.T, fake *remoteReadFiler, cancel chan struct{}) <-chan fuse.Status { + t.Helper() + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, fake) + + root := util.FullPath("/") + wfs.inodeToPath.MarkChildrenCached(root) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 4096}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 4096}, + }, 0, 0) + + done := make(chan fuse.Status, 1) + go func() { + _, code := wfs.Read(cancel, &fuse.ReadIn{ + InHeader: fuse.InHeader{NodeId: inode}, + Fh: uint64(fh.fh), + Size: 4096, + }, make([]byte, 4096)) + done <- code + }() + return done +} + +// The kernel's FUSE contract answers an interrupted request with EINTR on the +// original request. Interrupts come from any handled signal, Go's preemption +// SIGURG included, so EIO here would fail healthy reads. +func TestReadInterruptedReturnsEINTR(t *testing.T) { + fake := &remoteReadFiler{stalled: true, started: make(chan struct{})} + cancel := make(chan struct{}) + done := startRemoteRead(t, fake, cancel) + + select { + case <-fake.started: + case <-time.After(10 * time.Second): + t.Fatal("read never reached the filer") + } + close(cancel) + + select { + case code := <-done: + if code != fuse.EINTR { + t.Fatalf("interrupted read returned %v, want EINTR", code) + } + case <-time.After(10 * time.Second): + t.Fatal("Read still blocked after the FUSE request was interrupted") + } +} + +// A read that fails without an interrupt is a real I/O error and stays EIO. +func TestReadFailureWithoutInterruptReturnsEIO(t *testing.T) { + fake := &remoteReadFiler{started: make(chan struct{})} + done := startRemoteRead(t, fake, make(chan struct{})) + + select { + case code := <-done: + if code != fuse.EIO { + t.Fatalf("failed read returned %v, want EIO", code) + } + select { + case <-fake.started: + default: + t.Fatal("read failed before it reached the filer") + } + case <-time.After(30 * time.Second): + t.Fatal("failed read never returned") + } +}