From 77a2b1b37843c513018ec132bfc551681537dcc6 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Mon, 10 Aug 2026 00:31:16 -0700
Subject: [PATCH] format: compute interior chunk cuts lazily

The cutter materialized every interior cut up front, so a tiny sidecar
declaring one enormous extent could allocate gigabytes of cut offsets
before any media byte arrived. Keep only the extent start offsets,
bounded by the extent count, and derive each cut arithmetically.
---
 weed/format/layout.go      | 40 +++++++++++++++++++++++---------------
 weed/format/layout_test.go | 27 +++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 16 deletions(-)

diff --git a/weed/format/layout.go b/weed/format/layout.go
index 8fefefa85..1ba4aeb10 100644
--- a/weed/format/layout.go
+++ b/weed/format/layout.go
@@ -151,9 +151,13 @@ func readUvarintBytes(reader *bytes.Reader, limit uint64) ([]byte, error) {
 
 // Cutter yields upload chunk boundaries: every extent boundary, plus
 // align-quantized cuts inside extents larger than maxChunkSize. A non-positive
-// maxChunkSize keeps each extent in one chunk.
+// maxChunkSize keeps each extent in one chunk. Interior cuts are computed
+// lazily, so memory stays bounded by the extent count no matter how large an
+// untrusted layout declares its extents to be.
 type Cutter struct {
-	cuts []int64 // absolute end offset of every chunk, ascending
+	starts  []int64 // extent start offsets, ascending
+	total   int64
+	quantum int64 // interior cut spacing; 0 = one chunk per extent
 }
 
 func (l *Layout) Cutter(maxChunkSize int64) *Cutter {
@@ -165,27 +169,31 @@ func (l *Layout) Cutter(maxChunkSize int64) *Cutter {
 			quantum = l.Align
 		}
 	}
-	var cuts []int64
+	starts := make([]int64, len(l.ExtentSizes))
 	var offset int64
-	for _, size := range l.ExtentSizes {
-		end := offset + size
-		if quantum > 0 {
-			for next := offset + quantum; next < end; next += quantum {
-				cuts = append(cuts, next)
-			}
-		}
-		cuts = append(cuts, end)
-		offset = end
+	for i, size := range l.ExtentSizes {
+		starts[i] = offset
+		offset += size
 	}
-	return &Cutter{cuts: cuts}
+	return &Cutter{starts: starts, total: offset, quantum: quantum}
 }
 
 // NextChunkSize returns the size of the chunk starting at offset, or 0 past
 // the end. It satisfies the filer upload loop's ChunkBoundaries interface.
 func (c *Cutter) NextChunkSize(offset int64) int64 {
-	i := sort.Search(len(c.cuts), func(i int) bool { return c.cuts[i] > offset })
-	if i == len(c.cuts) {
+	if offset < 0 || offset >= c.total {
 		return 0
 	}
-	return c.cuts[i] - offset
+	i := sort.Search(len(c.starts), func(i int) bool { return c.starts[i] > offset }) - 1
+	end := c.total
+	if i+1 < len(c.starts) {
+		end = c.starts[i+1]
+	}
+	remaining := end - offset
+	if c.quantum > 0 {
+		if step := c.quantum - (offset-c.starts[i])%c.quantum; step < remaining {
+			return step
+		}
+	}
+	return remaining
 }
diff --git a/weed/format/layout_test.go b/weed/format/layout_test.go
index 4256ba624..81b25e3a7 100644
--- a/weed/format/layout_test.go
+++ b/weed/format/layout_test.go
@@ -162,3 +162,30 @@ func TestCutterUnlimitedKeepsOneChunkPerExtent(t *testing.T) {
 		t.Fatalf("chunks = %v, want %v", chunks, want)
 	}
 }
+
+// A hostile layout may declare an enormous extent; the cutter must stay O(1)
+// per query instead of materializing every interior cut.
+func TestCutterHugeExtentStaysLazy(t *testing.T) {
+	const quantum = 4 << 20 // 4MiB, already a multiple of align 1
+	layout := &Layout{Format: "x", ExtentSizes: []int64{1 << 50, 188}, Align: 188}
+	cutter := layout.Cutter(quantum)
+	alignedQuantum := int64(quantum - quantum%188)
+	if got := cutter.NextChunkSize(0); got != alignedQuantum {
+		t.Fatalf("NextChunkSize(0) = %d, want %d", got, alignedQuantum)
+	}
+	if got := cutter.NextChunkSize(alignedQuantum * 1000); got != alignedQuantum {
+		t.Fatalf("mid-extent chunk = %d, want %d", got, alignedQuantum)
+	}
+	// the final interior chunk stops at the extent boundary
+	last := (int64(1<<50) / alignedQuantum) * alignedQuantum
+	if got := cutter.NextChunkSize(last); got != int64(1<<50)-last {
+		t.Fatalf("tail chunk = %d, want %d", got, int64(1<<50)-last)
+	}
+	// the next extent still cuts independently
+	if got := cutter.NextChunkSize(1 << 50); got != 188 {
+		t.Fatalf("second extent chunk = %d, want 188", got)
+	}
+	if got := cutter.NextChunkSize(1<<50 + 188); got != 0 {
+		t.Fatalf("past end = %d, want 0", got)
+	}
+}