filer: fail the request, not the process, when chunks cannot be resolved (#11685)

* filer: fail the request, not the process, when chunks cannot be resolved

CompactFileChunks and ViewFromChunks discarded the error from
NonOverlappingVisibleIntervals and iterated the nil interval list it
returns on failure. A CreateEntry or UpdateEntry whose context was
cancelled therefore panicked in SeparateGarbageChunks and took the whole
filer down, losing the metadata log it had not yet flushed.

Both functions now return the error. cleanupChunks fails the request
with it, the mount flush paths return it, the replication sinks and the
S3 SSE range reader return it, and a ChunkStreamReader that cannot build
its view fails every read with it and reports it as its SourceError
instead of reading as an empty file.

Fixes #11682

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* filer: propagate chunk resolution failures in scan paths

NonOverlappingVisibleIntervals can fail without a cancelled context; the
query engine and log-store callers discarded the error and dereferenced
nil intervals.

* query: propagate chunk resolution errors instead of skipping files

When a manifest cannot be resolved, ReadParquetStatistics,
countLiveLogRowsExcludingParquetSources, and computeLiveLogMinMax logged a
warning and skipped the failed file, so queries returned understated counts
and wrong min/max values. Surface the errors instead: the fast-path
aggregation collector now returns a DataSourceError so the engine falls
back to a full scan, and the row-count helpers return the error rather
than a partial total.

---------

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/filer/filechunk_manifest_test.go

@@ -338,7 +338,10 @@ func TestCompactResolvedOverlappingManifests(t *testing.T) {
 	}
 
 	// CompactFileChunks on resolved sub-chunks (nil lookupFn is fine for non-manifest chunks)
-	compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
+	compacted, garbage, err := CompactFileChunks(context.Background(), nil, dataChunks)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if len(compacted) != 3 {
 		t.Fatalf("expected 3 compacted chunks, got %d", len(compacted))
@@ -487,7 +490,10 @@ func TestCompactMultipleOverlappingManifestGenerations(t *testing.T) {
 	}
 
 	// Compact: only generation 5 (the newest 3 chunks) should survive
-	compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
+	compacted, garbage, err := CompactFileChunks(context.Background(), nil, dataChunks)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if len(compacted) != 3 {
 		t.Errorf("expected 3 compacted (gen 5), got %d", len(compacted))
 	}

weed/filer/filechunks.go

@@ -63,9 +63,15 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 	return finalETag
 }
 
-func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
+// CompactFileChunks splits chunks into those still visible and those fully
+// covered by newer writes. It fails if the chunk manifests cannot be resolved,
+// including when ctx is cancelled.
+func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk, err error) {
 
-	visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
+	visibles, err := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
+	if err != nil {
+		return nil, nil, err
+	}
 
 	compacted, garbage = SeparateGarbageChunks(visibles, chunks)
 
@@ -194,11 +200,17 @@ func (cv *ChunkView) CanRangeFetch() bool {
 	return cv.CipherKey == nil && !cv.IsGzipped
 }
 
-func ViewFromChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+// ViewFromChunks returns the views of chunks that cover [offset, offset+size).
+// It fails if the chunk manifests cannot be resolved, including when ctx is
+// cancelled.
+func ViewFromChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView], err error) {
 
-	visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, offset, offset+size)
+	visibles, err := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, offset, offset+size)
+	if err != nil {
+		return nil, err
+	}
 
-	return ViewFromVisibleIntervals(visibles, offset, size)
+	return ViewFromVisibleIntervals(visibles, offset, size), nil
 
 }
 

weed/filer/filechunks2_test.go

@@ -66,7 +66,10 @@ func TestCompactFileChunksRealCase(t *testing.T) {
 
 	printChunks("before", chunks)
 
-	compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
+	compacted, garbage, err := CompactFileChunks(context.Background(), nil, chunks)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	printChunks("compacted", compacted)
 	printChunks("garbage", garbage)

weed/filer/filechunks_cancel_test.go

@@ -0,0 +1,58 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// cancelledChunks returns a cancelled context and plain data chunks. Resolving
+// chunks under a cancelled context fails even without manifests, so these
+// exercise the error path with no lookup function.
+func cancelledChunks() (context.Context, []*filer_pb.FileChunk) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	return ctx, []*filer_pb.FileChunk{
+		{FileId: "1,01", Offset: 0, Size: 100, ModifiedTsNs: 1},
+		{FileId: "1,02", Offset: 0, Size: 100, ModifiedTsNs: 2},
+	}
+}
+
+func TestCompactFileChunksCancelledContext(t *testing.T) {
+	ctx, chunks := cancelledChunks()
+
+	compacted, garbage, err := CompactFileChunks(ctx, nil, chunks)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("err = %v, want context.Canceled", err)
+	}
+	if compacted != nil || garbage != nil {
+		t.Fatalf("compacted = %v, garbage = %v, want both nil on error", compacted, garbage)
+	}
+}
+
+func TestViewFromChunksCancelledContext(t *testing.T) {
+	ctx, chunks := cancelledChunks()
+
+	views, err := ViewFromChunks(ctx, nil, chunks, 0, 100)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("err = %v, want context.Canceled", err)
+	}
+	if views != nil {
+		t.Fatalf("views = %v, want nil on error", views)
+	}
+}
+
+func TestChunkStreamReaderCancelledContext(t *testing.T) {
+	ctx, chunks := cancelledChunks()
+
+	reader := NewChunkStreamReaderFromLookup(ctx, nil, chunks)
+	n, err := reader.Read(make([]byte, 10))
+	if n != 0 || !errors.Is(err, context.Canceled) {
+		t.Fatalf("Read = %d, %v, want 0, context.Canceled", n, err)
+	}
+	if !errors.Is(reader.SourceError(), context.Canceled) {
+		t.Fatalf("SourceError = %v, want context.Canceled", reader.SourceError())
+	}
+}

weed/filer/filechunks_test.go

@@ -22,7 +22,10 @@ func TestCompactFileChunks(t *testing.T) {
 		{Offset: 110, Size: 200, FileId: "jkl", ModifiedTsNs: 300},
 	}
 
-	compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
+	compacted, garbage, err := CompactFileChunks(context.Background(), nil, chunks)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if len(compacted) != 3 {
 		t.Fatalf("unexpected compacted: %d", len(compacted))
@@ -55,7 +58,10 @@ func TestCompactFileChunks2(t *testing.T) {
 		})
 	}
 
-	compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
+	compacted, garbage, err := CompactFileChunks(context.Background(), nil, chunks)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if len(compacted) != 4 {
 		t.Fatalf("unexpected compacted: %d", len(compacted))
@@ -427,7 +433,10 @@ func TestChunksReading(t *testing.T) {
 			// continue
 		}
 		log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i)
-		chunks := ViewFromChunks(context.Background(), nil, testcase.Chunks, testcase.Offset, testcase.Size)
+		chunks, err := ViewFromChunks(context.Background(), nil, testcase.Chunks, testcase.Offset, testcase.Size)
+		if err != nil {
+			t.Fatal(err)
+		}
 		x := -1
 		for c := chunks.Front(); c != nil; c = c.Next {
 			x++
@@ -474,7 +483,9 @@ func BenchmarkCompactFileChunks(b *testing.B) {
 	}
 
 	for n := 0; n < b.N; n++ {
-		CompactFileChunks(context.Background(), nil, chunks)
+		if _, _, err := CompactFileChunks(context.Background(), nil, chunks); err != nil {
+			b.Fatal(err)
+		}
 	}
 }
 
@@ -563,7 +574,10 @@ func TestCompactFileChunks3(t *testing.T) {
 		{Offset: 300, Size: 100, FileId: "def", ModifiedTsNs: 200},
 	}
 
-	compacted, _ := CompactFileChunks(context.Background(), nil, chunks)
+	compacted, _, err := CompactFileChunks(context.Background(), nil, chunks)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if len(compacted) != 4 {
 		t.Fatalf("unexpected compacted: %d", len(compacted))

weed/filer/stream.go

@@ -386,9 +386,18 @@ var _ = io.ReadSeeker(&ChunkStreamReader{})
 var _ = io.ReaderAt(&ChunkStreamReader{})
 var _ = io.Closer(&ChunkStreamReader{})
 
+// doNewChunkStreamReader builds a reader over chunks. If their manifests cannot
+// be resolved, the reader fails every read with that error and reports it as its
+// SourceError.
 func doNewChunkStreamReader(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
-	chunkViews := ViewFromChunks(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
+	chunkViews, err := ViewFromChunks(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
+	if err != nil {
+		return &ChunkStreamReader{
+			lookupFileId: lookupFileIdFn,
+			sourceErr:    err,
+		}
+	}
 
 	var totalSize int64
 	for x := chunkViews.Front(); x != nil; x = x.Next {
@@ -487,6 +496,10 @@ func insideChunk(offset int64, chunk *ChunkView) bool {
 }
 
 func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
+	// a reader without chunk views has a source error only if ViewFromChunks failed
+	if c.head == nil && c.sourceErr != nil {
+		return c.sourceErr
+	}
 	// stay in the same chunk
 	if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
 		return nil