seaweedfs/weed/format/layout.go

package format

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"sort"
)

const (
	layoutVersion = 1

	// MaxExtentCount bounds decoded layouts; it also bounds the chunk count a
	// layout can force on an entry.
	MaxExtentCount = 1 << 20
	// MaxPayloadBytes bounds the adapter payload carried in entry metadata.
	MaxPayloadBytes = 16 << 20
)

// TotalSize returns the file size the layout describes.
func (l *Layout) TotalSize() int64 {
	var total int64
	for _, size := range l.ExtentSizes {
		total += size
	}
	return total
}

// ExtentRange returns the byte range of extent i.
func (l *Layout) ExtentRange(i int) (offset, size int64, ok bool) {
	if i < 0 || i >= len(l.ExtentSizes) {
		return 0, 0, false
	}
	for _, extentSize := range l.ExtentSizes[:i] {
		offset += extentSize
	}
	return offset, l.ExtentSizes[i], true
}

// Validate checks layout consistency. A negative fileSize skips the total size
// check.
func (l *Layout) Validate(fileSize int64) error {
	if l.Format == "" {
		return fmt.Errorf("layout has no format name")
	}
	if l.Align < 1 {
		return fmt.Errorf("layout align %d is invalid", l.Align)
	}
	if len(l.ExtentSizes) == 0 {
		return fmt.Errorf("layout has no extents")
	}
	if len(l.ExtentSizes) > MaxExtentCount {
		return fmt.Errorf("layout has too many extents: %d", len(l.ExtentSizes))
	}
	if len(l.Payload) > MaxPayloadBytes {
		return fmt.Errorf("layout payload is too large: %d bytes", len(l.Payload))
	}
	var total int64
	for i, size := range l.ExtentSizes {
		if size <= 0 {
			return fmt.Errorf("extent %d has invalid size %d", i, size)
		}
		if size > math.MaxInt64-total {
			return fmt.Errorf("extent %d overflows the file size", i)
		}
		total += size
	}
	if fileSize >= 0 && total != fileSize {
		return fmt.Errorf("layout describes %d bytes but the file has %d", total, fileSize)
	}
	return nil
}

// Encode serializes the layout for the LayoutKey extended attribute.
func (l *Layout) Encode() ([]byte, error) {
	if err := l.Validate(-1); err != nil {
		return nil, err
	}
	out := []byte{layoutVersion}
	out = binary.AppendUvarint(out, uint64(len(l.Format)))
	out = append(out, l.Format...)
	out = binary.AppendUvarint(out, uint64(l.Align))
	out = binary.AppendUvarint(out, uint64(len(l.ExtentSizes)))
	for _, size := range l.ExtentSizes {
		out = binary.AppendUvarint(out, uint64(size))
	}
	out = binary.AppendUvarint(out, uint64(len(l.Payload)))
	out = append(out, l.Payload...)
	return out, nil
}

// DecodeLayout parses an encoded layout and validates it.
func DecodeLayout(data []byte) (*Layout, error) {
	reader := bytes.NewReader(data)
	version, err := reader.ReadByte()
	if err != nil || version != layoutVersion {
		return nil, fmt.Errorf("unsupported layout version")
	}
	name, err := readUvarintBytes(reader, 256)
	if err != nil {
		return nil, fmt.Errorf("read layout format: %w", err)
	}
	align, err := binary.ReadUvarint(reader)
	if err != nil || align > math.MaxInt64 {
		return nil, fmt.Errorf("read layout align: invalid")
	}
	count, err := binary.ReadUvarint(reader)
	if err != nil || count > MaxExtentCount {
		return nil, fmt.Errorf("read layout extent count: invalid")
	}
	sizes := make([]int64, count)
	for i := range sizes {
		size, err := binary.ReadUvarint(reader)
		if err != nil || size > math.MaxInt64 {
			return nil, fmt.Errorf("read extent %d size: invalid", i)
		}
		sizes[i] = int64(size)
	}
	payload, err := readUvarintBytes(reader, MaxPayloadBytes)
	if err != nil {
		return nil, fmt.Errorf("read layout payload: %w", err)
	}
	if reader.Len() != 0 {
		return nil, fmt.Errorf("layout has %d trailing bytes", reader.Len())
	}
	layout := &Layout{Format: string(name), ExtentSizes: sizes, Align: int64(align), Payload: payload}
	if err := layout.Validate(-1); err != nil {
		return nil, err
	}
	return layout, nil
}

func readUvarintBytes(reader *bytes.Reader, limit uint64) ([]byte, error) {
	length, err := binary.ReadUvarint(reader)
	if err != nil {
		return nil, err
	}
	if length > limit || length > uint64(reader.Len()) {
		return nil, fmt.Errorf("length %d is out of bounds", length)
	}
	if length == 0 {
		return nil, nil
	}
	data := make([]byte, length)
	if _, err := reader.Read(data); err != nil {
		return nil, err
	}
	return data, nil
}

// Cutter yields upload chunk boundaries: every extent boundary, plus
// align-quantized cuts inside extents larger than maxChunkSize. A non-positive
// maxChunkSize keeps each extent in one chunk.
type Cutter struct {
	cuts []int64 // absolute end offset of every chunk, ascending
}

func (l *Layout) Cutter(maxChunkSize int64) *Cutter {
	quantum := maxChunkSize
	if quantum > 0 && l.Align > 1 {
		quantum -= quantum % l.Align
		if quantum <= 0 {
			// An align larger than the chunk limit still cuts on whole atoms.
			quantum = l.Align
		}
	}
	var cuts []int64
	var offset int64
	for _, size := range l.ExtentSizes {
		end := offset + size
		if quantum > 0 {
			for next := offset + quantum; next < end; next += quantum {
				cuts = append(cuts, next)
			}
		}
		cuts = append(cuts, end)
		offset = end
	}
	return &Cutter{cuts: cuts}
}

// NextChunkSize returns the size of the chunk starting at offset, or 0 past
// the end. It satisfies the filer upload loop's ChunkBoundaries interface.
func (c *Cutter) NextChunkSize(offset int64) int64 {
	i := sort.Search(len(c.cuts), func(i int) bool { return c.cuts[i] > offset })
	if i == len(c.cuts) {
		return 0
	}
	return c.cuts[i] - offset
}