diff --git a/weed/format/layout.go b/weed/format/layout.go index 8fefefa85..1ba4aeb10 100644 --- a/weed/format/layout.go +++ b/weed/format/layout.go @@ -151,9 +151,13 @@ func readUvarintBytes(reader *bytes.Reader, limit uint64) ([]byte, error) { // Cutter yields upload chunk boundaries: every extent boundary, plus // align-quantized cuts inside extents larger than maxChunkSize. A non-positive -// maxChunkSize keeps each extent in one chunk. +// maxChunkSize keeps each extent in one chunk. Interior cuts are computed +// lazily, so memory stays bounded by the extent count no matter how large an +// untrusted layout declares its extents to be. type Cutter struct { - cuts []int64 // absolute end offset of every chunk, ascending + starts []int64 // extent start offsets, ascending + total int64 + quantum int64 // interior cut spacing; 0 = one chunk per extent } func (l *Layout) Cutter(maxChunkSize int64) *Cutter { @@ -165,27 +169,31 @@ func (l *Layout) Cutter(maxChunkSize int64) *Cutter { quantum = l.Align } } - var cuts []int64 + starts := make([]int64, len(l.ExtentSizes)) var offset int64 - for _, size := range l.ExtentSizes { - end := offset + size - if quantum > 0 { - for next := offset + quantum; next < end; next += quantum { - cuts = append(cuts, next) - } - } - cuts = append(cuts, end) - offset = end + for i, size := range l.ExtentSizes { + starts[i] = offset + offset += size } - return &Cutter{cuts: cuts} + return &Cutter{starts: starts, total: offset, quantum: quantum} } // NextChunkSize returns the size of the chunk starting at offset, or 0 past // the end. It satisfies the filer upload loop's ChunkBoundaries interface. func (c *Cutter) NextChunkSize(offset int64) int64 { - i := sort.Search(len(c.cuts), func(i int) bool { return c.cuts[i] > offset }) - if i == len(c.cuts) { + if offset < 0 || offset >= c.total { return 0 } - return c.cuts[i] - offset + i := sort.Search(len(c.starts), func(i int) bool { return c.starts[i] > offset }) - 1 + end := c.total + if i+1 < len(c.starts) { + end = c.starts[i+1] + } + remaining := end - offset + if c.quantum > 0 { + if step := c.quantum - (offset-c.starts[i])%c.quantum; step < remaining { + return step + } + } + return remaining } diff --git a/weed/format/layout_test.go b/weed/format/layout_test.go index 4256ba624..81b25e3a7 100644 --- a/weed/format/layout_test.go +++ b/weed/format/layout_test.go @@ -162,3 +162,30 @@ func TestCutterUnlimitedKeepsOneChunkPerExtent(t *testing.T) { t.Fatalf("chunks = %v, want %v", chunks, want) } } + +// A hostile layout may declare an enormous extent; the cutter must stay O(1) +// per query instead of materializing every interior cut. +func TestCutterHugeExtentStaysLazy(t *testing.T) { + const quantum = 4 << 20 // 4MiB, already a multiple of align 1 + layout := &Layout{Format: "x", ExtentSizes: []int64{1 << 50, 188}, Align: 188} + cutter := layout.Cutter(quantum) + alignedQuantum := int64(quantum - quantum%188) + if got := cutter.NextChunkSize(0); got != alignedQuantum { + t.Fatalf("NextChunkSize(0) = %d, want %d", got, alignedQuantum) + } + if got := cutter.NextChunkSize(alignedQuantum * 1000); got != alignedQuantum { + t.Fatalf("mid-extent chunk = %d, want %d", got, alignedQuantum) + } + // the final interior chunk stops at the extent boundary + last := (int64(1<<50) / alignedQuantum) * alignedQuantum + if got := cutter.NextChunkSize(last); got != int64(1<<50)-last { + t.Fatalf("tail chunk = %d, want %d", got, int64(1<<50)-last) + } + // the next extent still cuts independently + if got := cutter.NextChunkSize(1 << 50); got != 188 { + t.Fatalf("second extent chunk = %d, want 188", got) + } + if got := cutter.NextChunkSize(1<<50 + 188); got != 0 { + t.Fatalf("past end = %d, want 0", got) + } +}