mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
generate vtproto marshalers for filer_pb and use them on the metadata log path (#10337)
* generate vtproto marshalers for filer_pb and use them on the metadata log path Reflection-based proto.Unmarshal allocates a fresh message tree through reflect.New on every call. On the metadata subscription fan-out the same event is decoded once per subscriber, so reflect.New tops the decode churn under many mounts. Generate MarshalVT/UnmarshalVT/SizeVT for filer.proto (a separate filer_vtproto.pb.go, filer.pb.go untouched) and call them on the log entry marshal and the subscribe/replay decode paths. UnmarshalVT allocates message structs directly and copies byte and string fields, so it stays wire-compatible with proto.Unmarshal and preserves the non-aliasing the persisted-log cache depends on. For SubscribeMetadataResponse this cuts decode allocations 69 -> 50 and ~4.5us -> ~2.1us per event; the win scales with subscriber overlap. * marshal log entries directly into the buffer SizeVT is allocation-free and MarshalToSizedBufferVT writes into a pre-sized slice, so the log entry can be marshaled straight into logBuffer.buf. This drops the per-entry MarshalVT allocation and the follow-up copy on the write path. * expand vtproto benchmarks: marshal, decode, and marshal-into-buffer by chunk count Parametrize by nested-message count (chunks per event) and add encode + zero-alloc marshal-into-buffer benchmarks alongside the decode one, so the write-path win from MarshalToSizedBufferVT is measurable too. * keep proto.Unmarshal for metadata events to preserve UTF-8 validation UnmarshalVT skips proto3's UTF-8 validation of string fields, so a SubscribeMetadataResponse with an invalid-UTF-8 string (e.g. Directory "\xff") that proto.Unmarshal rejects would decode and reach path filtering and subscribers. Decode events with proto.Unmarshal again; UnmarshalVT stays on the log entry paths, whose only variable-length fields are bytes and so carry no UTF-8 constraint. Tests cover the codec difference and that a malformed event is skipped before delivery.
This commit is contained in:
1 parent
0ad83d5061
commit
c015cc3939
11 files changed
+24628
-36
No files matched your search
@@ -8,13 +8,11 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
"golang.org/x/sync/singleflight"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -201,10 +199,10 @@ func loadLogFileEntries(masterClient *wdclient.MasterClient, chunk *filer_pb.Fil
|
||||
// decodeLogRecords parses size-prefixed LogEntry records. A buffer that stops
|
||||
// mid-record, or whose size prefix is garbage (also the symptom of starting
|
||||
// mid-record), reports errLogChunkIncomplete with the cleanly decoded prefix.
|
||||
// Since proto.Unmarshal is permissive enough to accept misaligned bytes,
|
||||
// Since UnmarshalVT is permissive enough to accept misaligned bytes,
|
||||
// records must also satisfy the writer's invariants: never empty, a positive
|
||||
// timestamp, and strictly increasing within one flushed buffer.
|
||||
// proto.Unmarshal copies all bytes, so the entries do not alias data.
|
||||
// UnmarshalVT copies all bytes, so the entries do not alias data.
|
||||
func decodeLogRecords(data []byte) (entries []*filer_pb.LogEntry, cacheable bool, err error) {
|
||||
var lastTsNs int64
|
||||
for pos := 0; pos < len(data); {
|
||||
@@ -220,7 +218,7 @@ func decodeLogRecords(data []byte) (entries []*filer_pb.LogEntry, cacheable bool
|
||||
return entries, false, errLogChunkIncomplete
|
||||
}
|
||||
logEntry := &filer_pb.LogEntry{}
|
||||
if unmarshalErr := proto.Unmarshal(data[pos+4:pos+4+size], logEntry); unmarshalErr != nil {
|
||||
if unmarshalErr := logEntry.UnmarshalVT(data[pos+4 : pos+4+size]); unmarshalErr != nil {
|
||||
return entries, false, errLogChunkIncomplete
|
||||
}
|
||||
if logEntry.TsNs <= lastTsNs {
|
||||
|
||||
Reference in new issue
Block a user