Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-13 18:10:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

[Mount] Add ChunkGroup seeking tests and fix boundary handling (#11223)

Browse Source 
* fix issue 11221

* reply ai comments
This commit is contained in:
ssshr-66
2026-09-08 10:50:41 -07:00
committed by GitHub
parent 213f4c5d5c
commit c6b330be2b
5 changed files with 264 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/filer/filechunk_group.go
+24 -20
View File
@@ -262,7 +262,6 @@ const (
// SEEK_HOLE uint32 = 4 // seek to next hole after the offset
)
// FIXME: needa tests
func (group *ChunkGroup) SearchChunks(ctx context.Context, offset, fileSize int64, whence uint32) (found bool, out int64) {
group.sectionsLock.RLock()
defer group.sectionsLock.RUnlock()
@@ -273,32 +272,37 @@ func (group *ChunkGroup) SearchChunks(ctx context.Context, offset, fileSize int6
func (group *ChunkGroup) doSearchChunks(ctx context.Context, offset, fileSize int64, whence uint32) (found bool, out int64) {
sectionIndex, maxSectionIndex := SectionIndex(offset/SectionSize), SectionIndex(fileSize/SectionSize)
if whence == SEEK_DATA {
for si := sectionIndex; si < maxSectionIndex+1; si++ {
section, foundSection := group.sections[si]
for si := sectionIndex; si <= maxSectionIndex; si++ {
sectionStart, sectionStop := sectionBounds(si, fileSize)
sectionStart = max(offset, sectionStart)
if sectionStart >= sectionStop {
continue
}
section, foundSection := group.sections[si]
if whence == SEEK_DATA {
if !foundSection {
continue
}
sectionStart := section.DataStartOffset(ctx, group, offset, fileSize)
if sectionStart == -1 {
continue
dataStart := section.DataStartOffset(ctx, group, sectionStart, fileSize)
if dataStart >= sectionStart && dataStart < sectionStop {
return true, dataStart
}
continue
}
// whence == SEEK_HOLE
if !foundSection {
return true, sectionStart
}
return false, 0
} else {
// whence == SEEK_HOLE
for si := sectionIndex; si < maxSectionIndex; si++ {
section, foundSection := group.sections[si]
if !foundSection {
return true, offset
}
holeStart := section.NextStopOffset(ctx, group, offset, fileSize)
if holeStart%SectionSize == 0 {
continue
}
holeStart := section.NextStopOffset(ctx, group, sectionStart, fileSize)
if holeStart < sectionStop {
return true, holeStart
}
return true, fileSize
}
if whence == SEEK_DATA {
return false, 0
}
return true, fileSize
}
weed/filer/filechunk_group_test.go
+187 -15
View File
@@ -4,9 +4,11 @@ import (
"context"
"errors"
"io"
"math"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/stretchr/testify/assert"
)
@@ -231,32 +233,202 @@ func TestChunkGroup_SearchChunks_Cancellation(t *testing.T) {
})
}
func TestChunkGroup_doSearchChunks(t *testing.T) {
type fields struct {
sections map[SectionIndex]*FileChunkSection
}
func TestChunkGroup_SearchChunks(t *testing.T) {
const seekHole uint32 = 4
type args struct {
offset int64
fileSize int64
whence uint32
}
tests := []struct {
name string
fields fields
args args
wantFound bool
wantOut int64
name string
chunks []*filer_pb.FileChunk
args args
wantFound bool
wantOffset int64
}{
// TODO: Add test cases.
{
name: "SEEK_DATA starts at the first data range after a hole",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 0, fileSize: 500, whence: SEEK_DATA},
wantFound: true,
wantOffset: 100,
},
{
name: "SEEK_DATA preserves an offset inside a data range",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 150, fileSize: 500, whence: SEEK_DATA},
wantFound: true,
wantOffset: 150,
},
{
name: "SEEK_DATA crosses a hole between data ranges",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 200, fileSize: 500, whence: SEEK_DATA},
wantFound: true,
wantOffset: 300,
},
{
name: "SEEK_DATA returns no match after the final data range",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 400, fileSize: 500, whence: SEEK_DATA},
wantFound: false,
wantOffset: 0,
},
{
name: "SEEK_HOLE starts at the sparse prefix",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 0, fileSize: 500, whence: seekHole},
wantFound: true,
wantOffset: 0,
},
{
name: "SEEK_HOLE finds the transition after data",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 150, fileSize: 500, whence: seekHole},
wantFound: true,
wantOffset: 200,
},
{
name: "SEEK_HOLE preserves an offset inside a hole",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 250, fileSize: 500, whence: seekHole},
wantFound: true,
wantOffset: 250,
},
{
name: "SEEK_HOLE returns the implicit trailing hole",
chunks: []*filer_pb.FileChunk{
{FileId: "data-1", Offset: 100, Size: 100},
{FileId: "data-2", Offset: 300, Size: 100},
},
args: args{offset: 400, fileSize: 500, whence: seekHole},
wantFound: true,
wantOffset: 400,
},
{
name: "SEEK_DATA at EOF has no match",
chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
args: args{offset: 500, fileSize: 500, whence: SEEK_DATA},
wantFound: false,
wantOffset: 0,
},
{
name: "SEEK_HOLE at EOF returns EOF",
chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
args: args{offset: 500, fileSize: 500, whence: seekHole},
wantFound: true,
wantOffset: 500,
},
{
name: "empty file has neither data nor a non-EOF hole",
chunks: nil,
args: args{offset: 0, fileSize: 0, whence: SEEK_DATA},
wantFound: false,
wantOffset: 0,
},
{
name: "empty file reports EOF for SEEK_HOLE",
chunks: nil,
args: args{offset: 0, fileSize: 0, whence: seekHole},
wantFound: true,
wantOffset: 0,
},
{
name: "SEEK_DATA crosses a section boundary",
chunks: []*filer_pb.FileChunk{
{FileId: "data-0", Offset: 0, Size: 16},
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
},
args: args{offset: 16, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
wantFound: true,
wantOffset: SectionSize - 16,
},
{
name: "SEEK_DATA finds data after a section boundary hole",
chunks: []*filer_pb.FileChunk{
{FileId: "data-0", Offset: 0, Size: 16},
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
},
args: args{offset: SectionSize, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
wantFound: true,
wantOffset: SectionSize + 16,
},
{
name: "SEEK_HOLE finds a hole at a section boundary",
chunks: []*filer_pb.FileChunk{
{FileId: "data-0", Offset: 0, Size: 16},
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
},
args: args{offset: SectionSize - 16, fileSize: 2*SectionSize + 32, whence: seekHole},
wantFound: true,
wantOffset: SectionSize,
},
{
name: "SEEK_HOLE finds a missing section",
chunks: []*filer_pb.FileChunk{
{FileId: "data-0", Offset: 0, Size: 16},
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
},
args: args{offset: 2 * SectionSize, fileSize: 2*SectionSize + 32, whence: seekHole},
wantFound: true,
wantOffset: 2 * SectionSize,
},
{
name: "SEEK_DATA finds data in the final section at MaxInt64 file size",
chunks: []*filer_pb.FileChunk{
{FileId: "final-data", Offset: math.MaxInt64 - 1, Size: 1},
},
args: args{offset: math.MaxInt64 - 1, fileSize: math.MaxInt64, whence: SEEK_DATA},
wantFound: true,
wantOffset: math.MaxInt64 - 1,
},
{
name: "SEEK_HOLE finds the final section hole at MaxInt64 file size",
chunks: []*filer_pb.FileChunk{
{FileId: "final-data", Offset: math.MaxInt64 - 2, Size: 1},
},
args: args{offset: math.MaxInt64 - 2, fileSize: math.MaxInt64, whence: seekHole},
wantFound: true,
wantOffset: math.MaxInt64 - 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
group := &ChunkGroup{
sections: tt.fields.sections,
group, err := NewChunkGroup(nil, nil, tt.chunks, 1, nil)
if !assert.NoError(t, err) {
return
}
gotFound, gotOut := group.doSearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence)
assert.Equalf(t, tt.wantFound, gotFound, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
assert.Equalf(t, tt.wantOut, gotOut, "doSearchChunks(%v, %v, %v)", tt.args.offset, tt.args.fileSize, tt.args.whence)
gotFound, gotOffset := group.SearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence)
assert.Equalf(t, tt.wantFound, gotFound, "SearchChunks(%v, %v, %v) found", tt.args.offset, tt.args.fileSize, tt.args.whence)
assert.Equalf(t, tt.wantOffset, gotOffset, "SearchChunks(%v, %v, %v) offset", tt.args.offset, tt.args.fileSize, tt.args.whence)
})
}
}
weed/filer/filechunk_section.go
+15 -4
View File
@@ -19,6 +19,15 @@ type FileChunkSection struct {
isPrepared bool
}
func sectionBounds(sectionIndex SectionIndex, fileSize int64) (start, stop int64) {
start = int64(sectionIndex) * SectionSize
stop = fileSize
if sectionIndex < SectionIndex(fileSize/SectionSize) {
stop = start + SectionSize
}
return
}
func NewFileChunkSection(si SectionIndex) *FileChunkSection {
return &FileChunkSection{
sectionIndex: si,
@@ -67,13 +76,15 @@ func (section *FileChunkSection) setupForRead(ctx context.Context, group *ChunkG
section.lock.Lock()
defer section.lock.Unlock()
sectionStart, sectionStop := sectionBounds(section.sectionIndex, fileSize)
if section.isPrepared {
section.reader.fileSize = fileSize
return
}
if section.visibleIntervals == nil {
section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
section.visibleIntervals = readResolvedChunks(section.chunks, sectionStart, sectionStop)
section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
if section.reader != nil {
_ = section.reader.Close()
@@ -81,11 +92,11 @@ func (section *FileChunkSection) setupForRead(ctx context.Context, group *ChunkG
}
}
if section.chunkViews == nil {
section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, sectionStart, sectionStop)
}
if section.reader == nil {
section.reader = NewChunkReaderAtFromClient(ctx, group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize), group.GetPrefetchCount())
section.reader = NewChunkReaderAtFromClient(ctx, group.readerCache, section.chunkViews, sectionStop, group.GetPrefetchCount())
}
section.isPrepared = true
@@ -113,7 +124,7 @@ func (section *FileChunkSection) DataStartOffset(ctx context.Context, group *Chu
continue
}
if offset < visible.start {
return offset
return visible.start
}
return offset
}
weed/mount/weedfs_file_lseek.go
+4 -5
View File
@@ -76,13 +76,12 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO
return fuse.OK
}
// in case we found no exact matches, we return the recommended fallbacks, that is:
// original offset for SEEK_DATA or end of file for an implicit hole
if in.Whence == SEEK_DATA {
out.Offset = in.Offset
} else {
out.Offset = uint64(fileSize)
return ENXIO
}
// If no explicit hole was found, the hole after the final data range is
// implicit and starts at the end of the file.
out.Offset = uint64(fileSize)
return fuse.OK
}
weed/mount/weedfs_file_lseek_test.go
+34
View File
@@ -0,0 +1,34 @@
package mount
import (
"testing"
"github.com/seaweedfs/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestLseekReturnsENXIOWhenNoDataFollowsOffset(t *testing.T) {
wfs, fh := newOpenFileHandle(t, 1000)
fh.SetEntry(&filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
FileMode: 0100644,
FileSize: 1000,
Inode: fh.inode,
},
Chunks: []*filer_pb.FileChunk{
{FileId: "data", Offset: 0, Size: 100},
},
})
in := &fuse.LseekIn{
Fh: uint64(fh.fh),
Offset: 100,
Whence: SEEK_DATA,
}
var out fuse.LseekOut
if status := wfs.Lseek(nil, in, &out); status != ENXIO {
t.Fatalf("Lseek(SEEK_DATA) status = %v, want ENXIO", status)
}
}