Files
seaweedfs/weed/pb/filer_pb/filer_pb_event_scan.go
T
Chris LuandClaude Opus 5.5 8e15110b4f filer: wake metadata subscribers only for directories they can match (#12669)
* log_buffer: let a subscriber skip wake-ups for unwanted partition keys

Every append woke every registered subscriber. SetSubscriberWakeFilter lets
a reader sleep through appends whose partition key it would filter out; it
still reads them on its next poll. Flushes and keyless appends wake everyone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer_pb: decide subscription matches from an event's directories

MetadataDirectoryMayMatchSubscription rules out events whose directory no
child path of can match the subscription, and ScanMetadataEventDirectories
reads an event's directory and new parent path without allocating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: wake subscribers of a move's destination directory

Appends are keyed by the source directory, so a filtered subscriber of the
destination would otherwise only see the move on its next poll.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: wake metadata subscribers only for directories they can match

Every metadata event woke and was scanned by every subscriber, so filer CPU
grew with the number of subscribers watching other paths. Readers now
register a directory wake filter, and the per-entry prefilter rules entries
out from their directories before the skeleton scan.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test that moves into a subscribed prefix pass the prefilter

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 20:53:41 -08:00

191 lines
4.6 KiB
Go

package filer_pb
import (
"google.golang.org/protobuf/encoding/protowire"
)
// ScanMetadataEventSkeleton extracts just the fields the subscription matcher
// reads, without materializing the entries' chunk lists. ok=false means the
// caller must fall back to a full unmarshal.
func ScanMetadataEventSkeleton(data []byte) (skeleton *SubscribeMetadataResponse, ok bool) {
skeleton = &SubscribeMetadataResponse{}
rest := data
for len(rest) > 0 {
num, typ, n := protowire.ConsumeTag(rest)
if n < 0 {
return nil, false
}
rest = rest[n:]
switch {
case num == 1 && typ == protowire.BytesType: // directory
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, false
}
skeleton.Directory = string(v)
rest = rest[m:]
case num == 2 && typ == protowire.BytesType: // event_notification
if skeleton.EventNotification != nil {
// repeated occurrences of a message field merge; punt to a full decode
return nil, false
}
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, false
}
notification, scanOk := scanEventNotificationSkeleton(v)
if !scanOk {
return nil, false
}
skeleton.EventNotification = notification
rest = rest[m:]
case num == 3 && typ == protowire.VarintType: // ts_ns
v, m := protowire.ConsumeVarint(rest)
if m < 0 {
return nil, false
}
skeleton.TsNs = int64(v)
rest = rest[m:]
default:
m := protowire.ConsumeFieldValue(num, typ, rest)
if m < 0 {
return nil, false
}
rest = rest[m:]
}
}
return skeleton, true
}
// ScanMetadataEventDirectories extracts an event's directory and its
// new_parent_path without allocating. ok=false means the data is malformed.
func ScanMetadataEventDirectories(data []byte) (directory, newParentPath []byte, ok bool) {
rest := data
for len(rest) > 0 {
num, typ, n := protowire.ConsumeTag(rest)
if n < 0 {
return nil, nil, false
}
rest = rest[n:]
if typ != protowire.BytesType || (num != 1 && num != 2) {
m := protowire.ConsumeFieldValue(num, typ, rest)
if m < 0 {
return nil, nil, false
}
rest = rest[m:]
continue
}
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, nil, false
}
rest = rest[m:]
if num == 1 { // directory
directory = v
continue
}
// event_notification: repeated occurrences merge, so the last
// new_parent_path wins
for len(v) > 0 {
fnum, ftyp, fn := protowire.ConsumeTag(v)
if fn < 0 {
return nil, nil, false
}
v = v[fn:]
fm := protowire.ConsumeFieldValue(fnum, ftyp, v)
if fm < 0 {
return nil, nil, false
}
if fnum == 4 && ftyp == protowire.BytesType { // new_parent_path
newParentPath, _ = protowire.ConsumeBytes(v)
}
v = v[fm:]
}
}
return directory, newParentPath, true
}
func scanEventNotificationSkeleton(data []byte) (*EventNotification, bool) {
notification := &EventNotification{}
rest := data
for len(rest) > 0 {
num, typ, n := protowire.ConsumeTag(rest)
if n < 0 {
return nil, false
}
rest = rest[n:]
switch {
case num == 1 && typ == protowire.BytesType: // old_entry
if notification.OldEntry != nil {
return nil, false
}
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, false
}
name, scanOk := scanEntryName(v)
if !scanOk {
return nil, false
}
notification.OldEntry = &Entry{Name: name}
rest = rest[m:]
case num == 2 && typ == protowire.BytesType: // new_entry
if notification.NewEntry != nil {
return nil, false
}
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, false
}
name, scanOk := scanEntryName(v)
if !scanOk {
return nil, false
}
notification.NewEntry = &Entry{Name: name}
rest = rest[m:]
case num == 4 && typ == protowire.BytesType: // new_parent_path
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, false
}
notification.NewParentPath = string(v)
rest = rest[m:]
default:
m := protowire.ConsumeFieldValue(num, typ, rest)
if m < 0 {
return nil, false
}
rest = rest[m:]
}
}
return notification, true
}
// scanEntryName walks the whole entry so field order does not matter.
func scanEntryName(data []byte) (string, bool) {
name := ""
rest := data
for len(rest) > 0 {
num, typ, n := protowire.ConsumeTag(rest)
if n < 0 {
return "", false
}
rest = rest[n:]
if num == 1 && typ == protowire.BytesType { // name; last occurrence wins like proto merge
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return "", false
}
name = string(v)
rest = rest[m:]
continue
}
m := protowire.ConsumeFieldValue(num, typ, rest)
if m < 0 {
return "", false
}
rest = rest[m:]
}
return name, true
}