filer: cut metadata-subscription allocation churn (issue #10253) (#10260)

* log_buffer: read only ts_ns during metadata-read binary search

readTs ran a full proto.Unmarshal of each LogEntry just to compare
timestamps during ReadFromBuffer's binary search, allocating fresh
slices for the data and key byte fields on every probe. Under
metadata-subscription fan-out this decode churn dominated allocations
and drove heavy GC, inflating filer RSS well past the live heap.

Scan the wire format for ts_ns (field 1) instead and skip the data/key
payloads without copying them. readTs is now zero-alloc; the binary
search no longer touches the event payloads at all.

* log_buffer: alias event payloads instead of copying them on delivery

The subscribe read loops decoded each LogEntry with a full proto.Unmarshal,
allocating fresh slices for the data and key byte fields on every entry
(protobuf consumeBytesNoZero). When many metadata subscribers each re-read
the in-memory log window under a reconnect storm, that per-entry copy was a
large share of allocation churn -- and for entries a subscriber filters out
by prefix, the copied payload was never even looked at.

Decode with a wire scan that points data/key at sub-slices of the source
buffer instead of copying. The aliased slices are valid only for the
duration of the eachLogEntryFn callback; every current subscriber either
re-decodes the event into its own struct or hands it to a synchronous grpc
Send, so none retain them. Per delivered entry the loop decode drops from
176 B / 2 allocs to zero.

weed/util/log_buffer/log_buffer.go

@@ -2,6 +2,7 @@ package log_buffer
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"sync"
@@ -936,18 +937,6 @@ var bufferPool = sync.Pool{
 	},
 }
 
-// logEntryPool reduces allocations in readTs which is called frequently during binary search
-var logEntryPool = sync.Pool{
-	New: func() interface{} {
-		return &filer_pb.LogEntry{}
-	},
-}
-
-// resetLogEntry clears a LogEntry for pool reuse
-func resetLogEntry(e *filer_pb.LogEntry) {
-	proto.Reset(e)
-}
-
 func copiedBytes(buf []byte) (copied *bytes.Buffer) {
 	copied = bufferPool.Get().(*bytes.Buffer)
 	copied.Reset()
@@ -970,19 +959,129 @@ func readTs(buf []byte, pos int) (size int, ts int64, err error) {
 
 	entryData := buf[pos+4 : pos+4+size]
 
-	// Use pooled LogEntry to avoid allocation on every call
-	logEntry := logEntryPool.Get().(*filer_pb.LogEntry)
-	defer func() {
-		resetLogEntry(logEntry)
-		logEntryPool.Put(logEntry)
-	}()
-
-	err = proto.Unmarshal(entryData, logEntry)
+	// Read only LogEntry.ts_ns rather than unmarshaling the whole entry. This
+	// runs on every binary-search probe in ReadFromBuffer; a full proto.Unmarshal
+	// there allocates fresh slices for the data/key byte fields on each call, which
+	// dominated allocation churn under metadata-subscription fan-out.
+	ts, err = readTsNs(entryData)
 	if err != nil {
-		// Return error instead of failing fast
-		// This allows caller to handle corruption gracefully
-		return 0, 0, fmt.Errorf("corrupted log buffer: failed to unmarshal LogEntry at pos %d, size %d: %w", pos, size, err)
+		return 0, 0, fmt.Errorf("corrupted log buffer at pos %d, size %d: %w", pos, size, err)
 	}
 
-	return size, logEntry.TsNs, nil
+	return size, ts, nil
+}
+
+// readTsNs scans a marshaled LogEntry and returns its ts_ns (field 1, varint)
+// without decoding the data/key byte fields. Fields serialize in number order,
+// so ts_ns is normally the first tag and this returns after one varint; the full
+// scan keeps it correct for any field order. A missing field 1 means the proto3
+// default, ts_ns == 0.
+func readTsNs(entryData []byte) (tsNs int64, err error) {
+	for i := 0; i < len(entryData); {
+		tag, n := binary.Uvarint(entryData[i:])
+		if n <= 0 {
+			return 0, fmt.Errorf("bad field tag at %d", i)
+		}
+		i += n
+		fieldNum := tag >> 3
+		switch tag & 0x7 { // wire type
+		case 0: // varint
+			v, m := binary.Uvarint(entryData[i:])
+			if m <= 0 {
+				return 0, fmt.Errorf("bad varint for field %d", fieldNum)
+			}
+			i += m
+			if fieldNum == 1 { // ts_ns
+				return int64(v), nil
+			}
+		case 1: // 64-bit
+			i += 8
+		case 2: // length-delimited: read length, skip payload without copying
+			l, m := binary.Uvarint(entryData[i:])
+			if m <= 0 {
+				return 0, fmt.Errorf("bad length for field %d", fieldNum)
+			}
+			i += m
+			if l > uint64(len(entryData)-i) {
+				return 0, fmt.Errorf("field %d length %d overruns buffer", fieldNum, l)
+			}
+			i += int(l)
+		case 5: // 32-bit
+			i += 4
+		default:
+			return 0, fmt.Errorf("unknown wire type for field %d", fieldNum)
+		}
+		if i > len(entryData) {
+			return 0, fmt.Errorf("field %d overruns buffer", fieldNum)
+		}
+	}
+	return 0, nil
+}
+
+// unmarshalLogEntryAliased decodes a marshaled LogEntry into out, pointing the
+// data and key fields at sub-slices of entryData instead of copying them. A full
+// proto.Unmarshal allocates fresh slices for those byte fields on every entry
+// (protobuf consumeBytesNoZero), which dominated allocation churn when many
+// metadata subscribers each re-read the in-memory log window.
+//
+// The aliased data/key are only valid while entryData is, i.e. for the duration
+// of the eachLogEntryFn callback. Callers must copy anything they retain past the
+// callback; all current subscribers do (they re-decode data into their own event
+// or hand it to a synchronous grpc Send).
+func unmarshalLogEntryAliased(entryData []byte, out *filer_pb.LogEntry) error {
+	out.TsNs = 0
+	out.PartitionKeyHash = 0
+	out.Data = nil
+	out.Key = nil
+	out.Offset = 0
+	for i := 0; i < len(entryData); {
+		tag, n := binary.Uvarint(entryData[i:])
+		if n <= 0 {
+			return fmt.Errorf("bad field tag at %d", i)
+		}
+		i += n
+		fieldNum := tag >> 3
+		switch tag & 0x7 { // wire type
+		case 0: // varint: ts_ns / partition_key_hash / offset
+			v, m := binary.Uvarint(entryData[i:])
+			if m <= 0 {
+				return fmt.Errorf("bad varint for field %d", fieldNum)
+			}
+			i += m
+			switch fieldNum {
+			case 1:
+				out.TsNs = int64(v)
+			case 2:
+				out.PartitionKeyHash = int32(v)
+			case 5:
+				out.Offset = int64(v)
+			}
+		case 2: // length-delimited: data / key, aliased not copied
+			l, m := binary.Uvarint(entryData[i:])
+			if m <= 0 {
+				return fmt.Errorf("bad length for field %d", fieldNum)
+			}
+			i += m
+			if l > uint64(len(entryData)-i) {
+				return fmt.Errorf("field %d length %d overruns buffer", fieldNum, l)
+			}
+			switch fieldNum {
+			case 3:
+				out.Data = entryData[i : i+int(l)]
+			case 4:
+				out.Key = entryData[i : i+int(l)]
+			}
+			i += int(l)
+		case 1: // 64-bit
+			i += 8
+		case 5: // 32-bit
+			i += 4
+		default:
+			return fmt.Errorf("unknown wire type for field %d", fieldNum)
+		}
+		if i > len(entryData) {
+			return fmt.Errorf("field %d overruns buffer", fieldNum)
+		}
+	}
+	return nil
 }