filer: fall back to range fetch when a chunk exceeds the reader budget

A ciphered or compressed chunk larger than readerCacheSizeMB can never
be read through the whole-chunk path — the budget rejects the buffer —
so its partial views must still range-fetch or the GET fails outright.

weed/filer/reader_at.go

@@ -355,8 +355,11 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 	// Fetching the chunk whole would multiply volume-server reads. Ciphered
 	// and compressed chunks are the exception: the volume server reads the
 	// whole blob to serve a range, so they take the shared whole-chunk path
-	// where one download serves every buffer.
-	if chunkView.CanRangeFetch() && (!chunkView.IsFullChunk() || c.readerPattern.IsRandomMode()) {
+	// where one download serves every buffer — unless the whole chunk cannot
+	// even fit the reader budget, in which case a range fetch is the only
+	// way to serve the request.
+	rangeFetch := chunkView.CanRangeFetch() || !c.readerCache.budget.canFit(int(chunkView.ChunkSize))
+	if rangeFetch && (!chunkView.IsFullChunk() || c.readerPattern.IsRandomMode()) {
 		c.readerCache.releaseStream(&c.stream)
 		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
 		if n > 0 {
@@ -386,7 +389,7 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
 // readChunkSliceAtForParallel is a simplified version for parallel chunk fetching
 // It doesn't update lastChunkFid or trigger prefetch (handled by the caller)
 func (c *ChunkReadAt) readChunkSliceAtForParallel(ctx context.Context, buffer []byte, chunkView *ChunkView, offset uint64) (n int, err error) {
-	if chunkView.CanRangeFetch() && !chunkView.IsFullChunk() {
+	if (chunkView.CanRangeFetch() || !c.readerCache.budget.canFit(int(chunkView.ChunkSize))) && !chunkView.IsFullChunk() {
 		n, err = c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
 		if n > 0 {
 			return n, err

weed/filer/reader_at_shared_test.go

@@ -367,6 +367,42 @@ func TestChunkReadAtRangeInsideOneChunkStaysRangeFetch(t *testing.T) {
 	}
 }
 
+// A compressed chunk larger than the reader cache budget can never be
+// downloaded whole — the budget rejects the buffer — so its partial view
+// must fall back to a range fetch even though each range costs a full
+// decompress server-side. The alternative is a failed GET.
+func TestChunkReadAtOversizedCompressedChunkFallsBackToRange(t *testing.T) {
+	const chunkSize = 1 << 20
+	const sliceSize = 16 << 10
+
+	budget := NewReaderCacheBudget(64 << 10) // smaller than the chunk
+	rc := NewReaderCache(64, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
+		return []string{"unused"}, nil
+	}, nil, budget)
+	defer rc.destroy()
+	fetches := fetchRecorder(rc)
+
+	views := NewIntervalList[*ChunkView]()
+	views.AppendInterval(&Interval[*ChunkView]{
+		StartOffset: 32 << 10,
+		StopOffset:  64 << 10,
+		Value:       &ChunkView{FileId: "chunk0", OffsetInChunk: 32 << 10, ViewSize: 32 << 10, ViewOffset: 32 << 10, ChunkSize: chunkSize, IsGzipped: true},
+	})
+
+	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize, 0)
+	buf := make([]byte, 32<<10)
+	if n, err := reader.ReadAt(buf, 32<<10); err != nil || n != len(buf) {
+		t.Fatalf("read: n=%d err=%v", n, err)
+	}
+
+	if len(*fetches) != 1 {
+		t.Fatalf("got %d fetches, want 1 range fetch: %+v", len(*fetches), *fetches)
+	}
+	if f := (*fetches)[0]; f.isFullChunk || f.offset != 32<<10 || f.size != 32<<10 {
+		t.Fatalf("fetch = %+v, want range fetch offset=%d size=%d", f, 32<<10, 32<<10)
+	}
+}
+
 // A chunk a stream is positioned in must outlast downloader-limit eviction:
 // otherwise a busy cache drops the buffer mid-stream and forces a refetch.
 func TestChunkReadAtPinnedChunkSurvivesEviction(t *testing.T) {

weed/filer/reader_cache.go

@@ -100,11 +100,12 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
 			// abort when slots are filled
 			return
 		}
-		if chunkView.CanRangeFetch() && !chunkView.IsFullChunk() {
+		if (chunkView.CanRangeFetch() || !rc.budget.canFit(int(chunkView.ChunkSize))) && !chunkView.IsFullChunk() {
 			// the view is clipped to part of the chunk and will be
 			// range-fetched, so prefetching it whole would download bytes
 			// nobody needs; a ciphered or compressed partial view needs
-			// the whole blob anyway and is worth prefetching
+			// the whole blob anyway and is worth prefetching, but not when
+			// it cannot fit the budget at all
 			continue
 		}
 

weed/filer/reader_cache_budget.go

@@ -93,6 +93,13 @@ func (b *ReaderCacheBudget) reserve(s *SingleChunkCacher) error {
 	}
 }
 
+// canFit reports whether a whole-chunk buffer of this size can ever be
+// reserved. A chunk bigger than the budget cannot be read through the
+// whole-chunk path at all, so callers must fall back to range fetches.
+func (b *ReaderCacheBudget) canFit(size int) bool {
+	return b == nil || int64(mem.AllocationSize(size)) <= b.limit
+}
+
 func (b *ReaderCacheBudget) complete(s *SingleChunkCacher) {
 	if b == nil {
 		return