mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-10 08:17:44 +02:00
Bruce Zou
eb6a7e93ca
* mount: fail reads with error when chunk manifest resolution fails When SetChunks fails to resolve a chunk manifest (e.g. the volume is on a remote tier with reads disabled), the sections map stays empty and readDataAtSequential/readDataAtParallel zero-fill every missing section as if it were a sparse hole. Reads then return all-zero data with no error, so a plain cp of a large manifest-based file silently produces a completely zero-filled file. Remember the resolve error in ChunkGroup (guarded by sectionsLock) and return it from ReadDataAt. A later successful SetChunks clears it. Fixes the mount path of #11286. * filer: propagate manifest resolve errors in streaming read paths ViewFromChunks discards the chunk manifest resolve error returned by NonOverlappingVisibleIntervals. On failure the chunk views come back empty, and the streaming paths zero-fill the entire requested range, serving HTTP 200 / WebDAV 200 responses whose body is all zeros. Propagate the error in PrepareStreamContentWithThrottler, PrepareStreamContentWithPrefetch and the WebDAV read path so these requests fail with 500 instead. Fixes the filer HTTP and WebDAV paths of #11286. * mount: fail lseek with EIO when chunk manifest resolution fails SearchChunks still consulted the stale section map after SetChunks recorded a manifest resolution failure, so SEEK_DATA/SEEK_HOLE would describe the unresolved regions as sparse holes or return ENXIO. Return the recorded error from SearchChunks and map it to EIO in Lseek. Also add regression tests for the stream preparation error paths. Addresses review feedback on #11287.
474 lines
17 KiB
Go
474 lines
17 KiB
Go
package filer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestChunkGroup_ReadDataAt_ErrorHandling(t *testing.T) {
|
|
// Test that ReadDataAt behaves correctly in various scenarios
|
|
// This indirectly verifies that our error handling fix works properly
|
|
|
|
// Create a ChunkGroup with no sections
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
t.Run("should return immediately on error", func(t *testing.T) {
|
|
// This test verifies that our fix is working by checking the behavior
|
|
// We'll create a simple scenario where the fix would make a difference
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
offset := int64(0)
|
|
|
|
// With an empty ChunkGroup, we should get no error
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
|
|
|
|
// Should return 100 (length of buffer) and no error since there are no sections
|
|
// and missing sections are filled with zeros
|
|
assert.Equal(t, 100, n)
|
|
assert.Equal(t, int64(0), tsNs)
|
|
assert.NoError(t, err)
|
|
|
|
// Verify buffer is filled with zeros
|
|
for i, b := range buff {
|
|
assert.Equal(t, byte(0), b, "buffer[%d] should be zero", i)
|
|
}
|
|
})
|
|
|
|
t.Run("should handle EOF correctly", func(t *testing.T) {
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(50) // File smaller than buffer
|
|
offset := int64(0)
|
|
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
|
|
|
|
// Should return 50 (file size) and no error
|
|
assert.Equal(t, 50, n)
|
|
assert.Equal(t, int64(0), tsNs)
|
|
assert.NoError(t, err)
|
|
})
|
|
|
|
t.Run("should return EOF when offset exceeds file size", func(t *testing.T) {
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(50)
|
|
offset := int64(100) // Offset beyond file size
|
|
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
|
|
|
|
assert.Equal(t, 0, n)
|
|
assert.Equal(t, int64(0), tsNs)
|
|
assert.Equal(t, io.EOF, err)
|
|
})
|
|
|
|
t.Run("should demonstrate the GitHub issue fix - errors should not be masked", func(t *testing.T) {
|
|
// This test demonstrates the exact scenario described in GitHub issue #6991
|
|
// where io.EOF could mask real errors if we continued processing sections
|
|
|
|
// The issue:
|
|
// - Before the fix: if section 1 returns a real error, but section 2 returns io.EOF,
|
|
// the real error would be overwritten by io.EOF
|
|
// - After the fix: return immediately on any error, preserving the original error
|
|
|
|
// Our fix ensures that we return immediately on ANY error (including io.EOF)
|
|
// This test verifies that the fix pattern works correctly for the most critical cases
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
|
|
// Test 1: Normal operation with no sections (filled with zeros)
|
|
n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, int64(0))
|
|
assert.Equal(t, 100, n, "should read full buffer")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero for missing sections")
|
|
assert.NoError(t, err, "should not error for missing sections")
|
|
|
|
// Test 2: Reading beyond file size should return io.EOF immediately
|
|
n, tsNs, err = group.ReadDataAt(context.Background(), fileSize, buff, fileSize+1)
|
|
assert.Equal(t, 0, n, "should not read any bytes when beyond file size")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
assert.Equal(t, io.EOF, err, "should return io.EOF when reading beyond file size")
|
|
|
|
// Test 3: Reading at exact file boundary
|
|
n, tsNs, err = group.ReadDataAt(context.Background(), fileSize, buff, fileSize)
|
|
assert.Equal(t, 0, n, "should not read any bytes at exact file size boundary")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
assert.Equal(t, io.EOF, err, "should return io.EOF at file boundary")
|
|
|
|
// The key insight: Our fix ensures that ANY error from section.readDataAt()
|
|
// causes immediate return with proper context (bytes read + timestamp + error)
|
|
// This prevents later sections from masking earlier errors, especially
|
|
// preventing io.EOF from masking network errors or other real failures.
|
|
})
|
|
|
|
t.Run("Context Cancellation", func(t *testing.T) {
|
|
// Test 4: Context cancellation should be properly propagated through ReadDataAt
|
|
|
|
// This test verifies that the context parameter is properly threaded through
|
|
// the call chain and that cancellation checks are in place at the right points
|
|
|
|
// Test with a pre-cancelled context to ensure the cancellation is detected
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // Cancel immediately
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
|
|
// Call ReadDataAt with the already cancelled context
|
|
n, tsNs, err := group.ReadDataAt(ctx, fileSize, buff, int64(0))
|
|
|
|
// For an empty ChunkGroup (no sections), the operation will complete successfully
|
|
// since it just fills the buffer with zeros. However, the important thing is that
|
|
// the context is properly threaded through the call chain.
|
|
// The actual cancellation would be more evident with real chunk sections that
|
|
// perform network operations.
|
|
|
|
if err != nil {
|
|
// If an error is returned, it should be a context cancellation error
|
|
assert.True(t,
|
|
errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
|
|
"Expected context.Canceled or context.DeadlineExceeded, got: %v", err)
|
|
} else {
|
|
// If no error (operation completed before cancellation check),
|
|
// verify normal behavior for empty ChunkGroup
|
|
assert.Equal(t, 100, n, "should read full buffer size when no sections exist")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
t.Log("Operation completed before context cancellation was checked - this is expected for empty ChunkGroup")
|
|
}
|
|
})
|
|
|
|
t.Run("Context Cancellation with Timeout", func(t *testing.T) {
|
|
// Test 5: Context with timeout should be respected
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
// Create a context with a very short timeout
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
|
|
defer cancel()
|
|
|
|
buff := make([]byte, 100)
|
|
fileSize := int64(1000)
|
|
|
|
// This should fail due to timeout
|
|
n, tsNs, err := group.ReadDataAt(ctx, fileSize, buff, int64(0))
|
|
|
|
// For this simple case with no sections, it might complete before timeout
|
|
// But if it does timeout, we should handle it properly
|
|
if err != nil {
|
|
assert.True(t,
|
|
errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
|
|
"Expected context.Canceled or context.DeadlineExceeded when context times out, got: %v", err)
|
|
} else {
|
|
// If no error, verify normal behavior
|
|
assert.Equal(t, 100, n, "should read full buffer size when no sections exist")
|
|
assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestChunkGroup_SearchChunks_Cancellation(t *testing.T) {
|
|
t.Run("Context Cancellation in SearchChunks", func(t *testing.T) {
|
|
// Test that SearchChunks properly handles context cancellation
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
// Test with a pre-cancelled context
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // Cancel immediately
|
|
|
|
fileSize := int64(1000)
|
|
offset := int64(0)
|
|
whence := uint32(3) // SEEK_DATA
|
|
|
|
// Call SearchChunks with cancelled context
|
|
found, resultOffset, _ := group.SearchChunks(ctx, offset, fileSize, whence)
|
|
|
|
// For an empty ChunkGroup, SearchChunks should complete quickly
|
|
// The main goal is to verify the context parameter is properly threaded through
|
|
// In real scenarios with actual chunk sections, context cancellation would be more meaningful
|
|
|
|
// Verify the function completes and returns reasonable values
|
|
assert.False(t, found, "should not find data in empty chunk group")
|
|
assert.Equal(t, int64(0), resultOffset, "should return 0 offset when no data found")
|
|
|
|
t.Log("SearchChunks completed with cancelled context - context threading verified")
|
|
})
|
|
|
|
t.Run("Context with Timeout in SearchChunks", func(t *testing.T) {
|
|
// Test SearchChunks with a timeout context
|
|
|
|
group := &ChunkGroup{
|
|
sections: make(map[SectionIndex]*FileChunkSection),
|
|
}
|
|
|
|
// Create a context with very short timeout
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
|
|
defer cancel()
|
|
|
|
fileSize := int64(1000)
|
|
offset := int64(0)
|
|
whence := uint32(3) // SEEK_DATA
|
|
|
|
// Call SearchChunks - should complete quickly for empty group
|
|
found, resultOffset, _ := group.SearchChunks(ctx, offset, fileSize, whence)
|
|
|
|
// Verify reasonable behavior
|
|
assert.False(t, found, "should not find data in empty chunk group")
|
|
assert.Equal(t, int64(0), resultOffset, "should return 0 offset when no data found")
|
|
})
|
|
}
|
|
|
|
func TestChunkGroup_SearchChunks(t *testing.T) {
|
|
const seekHole uint32 = 4
|
|
|
|
type args struct {
|
|
offset int64
|
|
fileSize int64
|
|
whence uint32
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
chunks []*filer_pb.FileChunk
|
|
args args
|
|
wantFound bool
|
|
wantOffset int64
|
|
}{
|
|
{
|
|
name: "SEEK_DATA starts at the first data range after a hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 0, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: 100,
|
|
},
|
|
{
|
|
name: "SEEK_DATA preserves an offset inside a data range",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 150, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: 150,
|
|
},
|
|
{
|
|
name: "SEEK_DATA crosses a hole between data ranges",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 200, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: 300,
|
|
},
|
|
{
|
|
name: "SEEK_DATA returns no match after the final data range",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 400, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: false,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE starts at the sparse prefix",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 0, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds the transition after data",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 150, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 200,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE preserves an offset inside a hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 250, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 250,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE returns the implicit trailing hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-1", Offset: 100, Size: 100},
|
|
{FileId: "data-2", Offset: 300, Size: 100},
|
|
},
|
|
args: args{offset: 400, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 400,
|
|
},
|
|
{
|
|
name: "SEEK_DATA at EOF has no match",
|
|
chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
|
|
args: args{offset: 500, fileSize: 500, whence: SEEK_DATA},
|
|
wantFound: false,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE at EOF returns EOF",
|
|
chunks: []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
|
|
args: args{offset: 500, fileSize: 500, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 500,
|
|
},
|
|
{
|
|
name: "empty file has neither data nor a non-EOF hole",
|
|
chunks: nil,
|
|
args: args{offset: 0, fileSize: 0, whence: SEEK_DATA},
|
|
wantFound: false,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "empty file reports EOF for SEEK_HOLE",
|
|
chunks: nil,
|
|
args: args{offset: 0, fileSize: 0, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 0,
|
|
},
|
|
{
|
|
name: "SEEK_DATA crosses a section boundary",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: 16, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: SectionSize - 16,
|
|
},
|
|
{
|
|
name: "SEEK_DATA finds data after a section boundary hole",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: SectionSize, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: SectionSize + 16,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds a hole at a section boundary",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: SectionSize - 16, fileSize: 2*SectionSize + 32, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: SectionSize,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds a missing section",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "data-0", Offset: 0, Size: 16},
|
|
{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
|
|
{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
|
|
},
|
|
args: args{offset: 2 * SectionSize, fileSize: 2*SectionSize + 32, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: 2 * SectionSize,
|
|
},
|
|
{
|
|
name: "SEEK_DATA finds data in the final section at MaxInt64 file size",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "final-data", Offset: math.MaxInt64 - 1, Size: 1},
|
|
},
|
|
args: args{offset: math.MaxInt64 - 1, fileSize: math.MaxInt64, whence: SEEK_DATA},
|
|
wantFound: true,
|
|
wantOffset: math.MaxInt64 - 1,
|
|
},
|
|
{
|
|
name: "SEEK_HOLE finds the final section hole at MaxInt64 file size",
|
|
chunks: []*filer_pb.FileChunk{
|
|
{FileId: "final-data", Offset: math.MaxInt64 - 2, Size: 1},
|
|
},
|
|
args: args{offset: math.MaxInt64 - 2, fileSize: math.MaxInt64, whence: seekHole},
|
|
wantFound: true,
|
|
wantOffset: math.MaxInt64 - 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
group, err := NewChunkGroup(nil, nil, tt.chunks, 1, nil)
|
|
if !assert.NoError(t, err) {
|
|
return
|
|
}
|
|
|
|
gotFound, gotOffset, err := group.SearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence)
|
|
assert.NoError(t, err)
|
|
assert.Equalf(t, tt.wantFound, gotFound, "SearchChunks(%v, %v, %v) found", tt.args.offset, tt.args.fileSize, tt.args.whence)
|
|
assert.Equalf(t, tt.wantOffset, gotOffset, "SearchChunks(%v, %v, %v) offset", tt.args.offset, tt.args.fileSize, tt.args.whence)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Regression test for silent zero-fill reads when chunk manifest resolution
|
|
// fails: ReadDataAt must return an error instead of treating the unresolved
|
|
// sections as sparse holes (https://github.com/seaweedfs/seaweedfs/issues/11286).
|
|
func TestChunkGroup_ReadDataAt_ManifestResolveFailure(t *testing.T) {
|
|
lookupErr := errors.New("lookup failed")
|
|
lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
|
|
return nil, lookupErr
|
|
}
|
|
|
|
chunks := []*filer_pb.FileChunk{
|
|
{FileId: "1,1679011dc64abd40", IsChunkManifest: true, Offset: 0, Size: 1 << 20},
|
|
}
|
|
|
|
group, err := NewChunkGroup(lookupFn, nil, chunks, 1, nil)
|
|
assert.Error(t, err, "manifest resolution should fail")
|
|
|
|
// Reads must fail with the resolve error, not silently return zeros.
|
|
buff := make([]byte, 16)
|
|
n, _, readErr := group.ReadDataAt(context.Background(), 1<<20, buff, 0)
|
|
assert.ErrorIs(t, readErr, lookupErr)
|
|
assert.Equal(t, 0, n)
|
|
|
|
// lseek (SEEK_DATA/SEEK_HOLE) must fail too, not misreport the whole
|
|
// file as sparse.
|
|
for _, whence := range []uint32{SEEK_DATA, 4 /* SEEK_HOLE */} {
|
|
found, _, seekErr := group.SearchChunks(context.Background(), 0, 1<<20, whence)
|
|
assert.ErrorIs(t, seekErr, lookupErr, "whence %d", whence)
|
|
assert.False(t, found, "whence %d", whence)
|
|
}
|
|
|
|
// A later successful SetChunks must clear the error.
|
|
err = group.SetChunks([]*filer_pb.FileChunk{
|
|
{FileId: "2,data", Offset: 0, Size: 16},
|
|
})
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, group.resolveErr)
|
|
}
|