diff --git a/weed/filer/filechunk_group.go b/weed/filer/filechunk_group.go index 0052ba707..d017d223a 100644 --- a/weed/filer/filechunk_group.go +++ b/weed/filer/filechunk_group.go @@ -262,7 +262,6 @@ const ( // SEEK_HOLE uint32 = 4 // seek to next hole after the offset ) -// FIXME: needa tests func (group *ChunkGroup) SearchChunks(ctx context.Context, offset, fileSize int64, whence uint32) (found bool, out int64) { group.sectionsLock.RLock() defer group.sectionsLock.RUnlock() @@ -273,32 +272,37 @@ func (group *ChunkGroup) SearchChunks(ctx context.Context, offset, fileSize int6 func (group *ChunkGroup) doSearchChunks(ctx context.Context, offset, fileSize int64, whence uint32) (found bool, out int64) { sectionIndex, maxSectionIndex := SectionIndex(offset/SectionSize), SectionIndex(fileSize/SectionSize) - if whence == SEEK_DATA { - for si := sectionIndex; si < maxSectionIndex+1; si++ { - section, foundSection := group.sections[si] + for si := sectionIndex; si <= maxSectionIndex; si++ { + sectionStart, sectionStop := sectionBounds(si, fileSize) + sectionStart = max(offset, sectionStart) + if sectionStart >= sectionStop { + continue + } + + section, foundSection := group.sections[si] + if whence == SEEK_DATA { if !foundSection { continue } - sectionStart := section.DataStartOffset(ctx, group, offset, fileSize) - if sectionStart == -1 { - continue + dataStart := section.DataStartOffset(ctx, group, sectionStart, fileSize) + if dataStart >= sectionStart && dataStart < sectionStop { + return true, dataStart } + continue + } + + // whence == SEEK_HOLE + if !foundSection { return true, sectionStart } - return false, 0 - } else { - // whence == SEEK_HOLE - for si := sectionIndex; si < maxSectionIndex; si++ { - section, foundSection := group.sections[si] - if !foundSection { - return true, offset - } - holeStart := section.NextStopOffset(ctx, group, offset, fileSize) - if holeStart%SectionSize == 0 { - continue - } + holeStart := section.NextStopOffset(ctx, group, sectionStart, fileSize) + if holeStart < sectionStop { return true, holeStart } - return true, fileSize } + + if whence == SEEK_DATA { + return false, 0 + } + return true, fileSize } diff --git a/weed/filer/filechunk_group_test.go b/weed/filer/filechunk_group_test.go index a7103ce2e..b04a61c83 100644 --- a/weed/filer/filechunk_group_test.go +++ b/weed/filer/filechunk_group_test.go @@ -4,9 +4,11 @@ import ( "context" "errors" "io" + "math" "testing" "time" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/stretchr/testify/assert" ) @@ -231,32 +233,202 @@ func TestChunkGroup_SearchChunks_Cancellation(t *testing.T) { }) } -func TestChunkGroup_doSearchChunks(t *testing.T) { - type fields struct { - sections map[SectionIndex]*FileChunkSection - } +func TestChunkGroup_SearchChunks(t *testing.T) { + const seekHole uint32 = 4 + type args struct { offset int64 fileSize int64 whence uint32 } tests := []struct { - name string - fields fields - args args - wantFound bool - wantOut int64 + name string + chunks []*filer_pb.FileChunk + args args + wantFound bool + wantOffset int64 }{ - // TODO: Add test cases. + { + name: "SEEK_DATA starts at the first data range after a hole", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 0, fileSize: 500, whence: SEEK_DATA}, + wantFound: true, + wantOffset: 100, + }, + { + name: "SEEK_DATA preserves an offset inside a data range", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 150, fileSize: 500, whence: SEEK_DATA}, + wantFound: true, + wantOffset: 150, + }, + { + name: "SEEK_DATA crosses a hole between data ranges", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 200, fileSize: 500, whence: SEEK_DATA}, + wantFound: true, + wantOffset: 300, + }, + { + name: "SEEK_DATA returns no match after the final data range", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 400, fileSize: 500, whence: SEEK_DATA}, + wantFound: false, + wantOffset: 0, + }, + { + name: "SEEK_HOLE starts at the sparse prefix", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 0, fileSize: 500, whence: seekHole}, + wantFound: true, + wantOffset: 0, + }, + { + name: "SEEK_HOLE finds the transition after data", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 150, fileSize: 500, whence: seekHole}, + wantFound: true, + wantOffset: 200, + }, + { + name: "SEEK_HOLE preserves an offset inside a hole", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 250, fileSize: 500, whence: seekHole}, + wantFound: true, + wantOffset: 250, + }, + { + name: "SEEK_HOLE returns the implicit trailing hole", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-1", Offset: 100, Size: 100}, + {FileId: "data-2", Offset: 300, Size: 100}, + }, + args: args{offset: 400, fileSize: 500, whence: seekHole}, + wantFound: true, + wantOffset: 400, + }, + { + name: "SEEK_DATA at EOF has no match", + chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}}, + args: args{offset: 500, fileSize: 500, whence: SEEK_DATA}, + wantFound: false, + wantOffset: 0, + }, + { + name: "SEEK_HOLE at EOF returns EOF", + chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}}, + args: args{offset: 500, fileSize: 500, whence: seekHole}, + wantFound: true, + wantOffset: 500, + }, + { + name: "empty file has neither data nor a non-EOF hole", + chunks: nil, + args: args{offset: 0, fileSize: 0, whence: SEEK_DATA}, + wantFound: false, + wantOffset: 0, + }, + { + name: "empty file reports EOF for SEEK_HOLE", + chunks: nil, + args: args{offset: 0, fileSize: 0, whence: seekHole}, + wantFound: true, + wantOffset: 0, + }, + { + name: "SEEK_DATA crosses a section boundary", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-0", Offset: 0, Size: 16}, + {FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16}, + {FileId: "data-1", Offset: SectionSize + 16, Size: 16}, + }, + args: args{offset: 16, fileSize: 2*SectionSize + 32, whence: SEEK_DATA}, + wantFound: true, + wantOffset: SectionSize - 16, + }, + { + name: "SEEK_DATA finds data after a section boundary hole", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-0", Offset: 0, Size: 16}, + {FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16}, + {FileId: "data-1", Offset: SectionSize + 16, Size: 16}, + }, + args: args{offset: SectionSize, fileSize: 2*SectionSize + 32, whence: SEEK_DATA}, + wantFound: true, + wantOffset: SectionSize + 16, + }, + { + name: "SEEK_HOLE finds a hole at a section boundary", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-0", Offset: 0, Size: 16}, + {FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16}, + {FileId: "data-1", Offset: SectionSize + 16, Size: 16}, + }, + args: args{offset: SectionSize - 16, fileSize: 2*SectionSize + 32, whence: seekHole}, + wantFound: true, + wantOffset: SectionSize, + }, + { + name: "SEEK_HOLE finds a missing section", + chunks: []*filer_pb.FileChunk{ + {FileId: "data-0", Offset: 0, Size: 16}, + {FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16}, + {FileId: "data-1", Offset: SectionSize + 16, Size: 16}, + }, + args: args{offset: 2 * SectionSize, fileSize: 2*SectionSize + 32, whence: seekHole}, + wantFound: true, + wantOffset: 2 * SectionSize, + }, + { + name: "SEEK_DATA finds data in the final section at MaxInt64 file size", + chunks: []*filer_pb.FileChunk{ + {FileId: "final-data", Offset: math.MaxInt64 - 1, Size: 1}, + }, + args: args{offset: math.MaxInt64 - 1, fileSize: math.MaxInt64, whence: SEEK_DATA}, + wantFound: true, + wantOffset: math.MaxInt64 - 1, + }, + { + name: "SEEK_HOLE finds the final section hole at MaxInt64 file size", + chunks: []*filer_pb.FileChunk{ + {FileId: "final-data", Offset: math.MaxInt64 - 2, Size: 1}, + }, + args: args{offset: math.MaxInt64 - 2, fileSize: math.MaxInt64, whence: seekHole}, + wantFound: true, + wantOffset: math.MaxInt64 - 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - group := &ChunkGroup{ - sections: tt.fields.sections, + group, err := NewChunkGroup(nil, nil, tt.chunks, 1, nil) + if !assert.NoError(t, err) { + return } - gotFound, gotOut := group.doSearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence) - assert.Equalf(t, tt.wantFound, gotFound, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence) - assert.Equalf(t, tt.wantOut, gotOut, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence) + + gotFound, gotOffset := group.SearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence) + assert.Equalf(t, tt.wantFound, gotFound, "SearchChunks(%v, %v, %v) found", tt.args.offset, tt.args.fileSize, tt.args.whence) + assert.Equalf(t, tt.wantOffset, gotOffset, "SearchChunks(%v, %v, %v) offset", tt.args.offset, tt.args.fileSize, tt.args.whence) }) } } diff --git a/weed/filer/filechunk_section.go b/weed/filer/filechunk_section.go index dd7c2ea48..204a5badd 100644 --- a/weed/filer/filechunk_section.go +++ b/weed/filer/filechunk_section.go @@ -19,6 +19,15 @@ type FileChunkSection struct { isPrepared bool } +func sectionBounds(sectionIndex SectionIndex, fileSize int64) (start, stop int64) { + start = int64(sectionIndex) * SectionSize + stop = fileSize + if sectionIndex < SectionIndex(fileSize/SectionSize) { + stop = start + SectionSize + } + return +} + func NewFileChunkSection(si SectionIndex) *FileChunkSection { return &FileChunkSection{ sectionIndex: si, @@ -67,13 +76,15 @@ func (section *FileChunkSection) setupForRead(ctx context.Context, group *ChunkG section.lock.Lock() defer section.lock.Unlock() + sectionStart, sectionStop := sectionBounds(section.sectionIndex, fileSize) + if section.isPrepared { section.reader.fileSize = fileSize return } if section.visibleIntervals == nil { - section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize) + section.visibleIntervals = readResolvedChunks(section.chunks, sectionStart, sectionStop) section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks) if section.reader != nil { _ = section.reader.Close() @@ -81,11 +92,11 @@ func (section *FileChunkSection) setupForRead(ctx context.Context, group *ChunkG } } if section.chunkViews == nil { - section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize) + section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, sectionStart, sectionStop) } if section.reader == nil { - section.reader = NewChunkReaderAtFromClient(ctx, group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize), group.GetPrefetchCount()) + section.reader = NewChunkReaderAtFromClient(ctx, group.readerCache, section.chunkViews, sectionStop, group.GetPrefetchCount()) } section.isPrepared = true @@ -113,7 +124,7 @@ func (section *FileChunkSection) DataStartOffset(ctx context.Context, group *Chu continue } if offset < visible.start { - return offset + return visible.start } return offset } diff --git a/weed/mount/weedfs_file_lseek.go b/weed/mount/weedfs_file_lseek.go index 9a1d1d2d7..826c1e0fc 100644 --- a/weed/mount/weedfs_file_lseek.go +++ b/weed/mount/weedfs_file_lseek.go @@ -76,13 +76,12 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO return fuse.OK } - // in case we found no exact matches, we return the recommended fallbacks, that is: - // original offset for SEEK_DATA or end of file for an implicit hole if in.Whence == SEEK_DATA { - out.Offset = in.Offset - } else { - out.Offset = uint64(fileSize) + return ENXIO } + // If no explicit hole was found, the hole after the final data range is + // implicit and starts at the end of the file. + out.Offset = uint64(fileSize) return fuse.OK } diff --git a/weed/mount/weedfs_file_lseek_test.go b/weed/mount/weedfs_file_lseek_test.go new file mode 100644 index 000000000..144cb28da --- /dev/null +++ b/weed/mount/weedfs_file_lseek_test.go @@ -0,0 +1,34 @@ +package mount + +import ( + "testing" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +func TestLseekReturnsENXIOWhenNoDataFollowsOffset(t *testing.T) { + wfs, fh := newOpenFileHandle(t, 1000) + fh.SetEntry(&filer_pb.Entry{ + Name: "file.txt", + Attributes: &filer_pb.FuseAttributes{ + FileMode: 0100644, + FileSize: 1000, + Inode: fh.inode, + }, + Chunks: []*filer_pb.FileChunk{ + {FileId: "data", Offset: 0, Size: 100}, + }, + }) + + in := &fuse.LseekIn{ + Fh: uint64(fh.fh), + Offset: 100, + Whence: SEEK_DATA, + } + var out fuse.LseekOut + + if status := wfs.Lseek(nil, in, &out); status != ENXIO { + t.Fatalf("Lseek(SEEK_DATA) status = %v, want ENXIO", status) + } +}