filer: wake metadata subscribers only for directories they can match (#12669)

* log_buffer: let a subscriber skip wake-ups for unwanted partition keys

Every append woke every registered subscriber. SetSubscriberWakeFilter lets
a reader sleep through appends whose partition key it would filter out; it
still reads them on its next poll. Flushes and keyless appends wake everyone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer_pb: decide subscription matches from an event's directories

MetadataDirectoryMayMatchSubscription rules out events whose directory no
child path of can match the subscription, and ScanMetadataEventDirectories
reads an event's directory and new parent path without allocating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: wake subscribers of a move's destination directory

Appends are keyed by the source directory, so a filtered subscriber of the
destination would otherwise only see the move on its next poll.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: wake metadata subscribers only for directories they can match

Every metadata event woke and was scanned by every subscriber, so filer CPU
grew with the number of subscribers watching other paths. Readers now
register a directory wake filter, and the per-entry prefilter rules entries
out from their directories before the skeleton scan.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test that moves into a subscribed prefix pass the prefilter

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-15 20:53:41 -08:00
1 parent f13b763552
commit 8e15110b4f
10 files changed
+414 -18

No files matched your search

+9
View File
@@ -288,6 +288,14 @@ func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFrom
}
}
// notifyRenameTarget wakes the subscribers of a move's destination directory,
// which the partition key (the source directory) does not cover.
func notifyRenameTarget(logBuffer *log_buffer.LogBuffer, event *filer_pb.SubscribeMetadataResponse) {
if target := filer_pb.MetadataEventTargetDirectory(event); target != event.Directory {
logBuffer.NotifySubscribers([]byte(target))
}
}
func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) {
data, err := proto.Marshal(event)
if err != nil {
@@ -300,6 +308,7 @@ func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetad
glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err)
return
}
notifyRenameTarget(f.LocalMetaLogBuffer, event)
// The in-memory copy now carries the position the stream assigns it, the
// same value subscribers will see delivered.
event.TsNs = tsNs
+40
View File
@@ -0,0 +1,40 @@
package filer
import (
"bytes"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
)
func TestNotifyRenameTargetWakesDestinationSubscribers(t *testing.T) {
logBuffer := log_buffer.NewLogBuffer("test", time.Minute, func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
defer logBuffer.ShutdownLogBuffer()
logBuffer.SetSubscriberWakeFilter("dst", func(key []byte) bool { return bytes.Equal(key, []byte("/dst")) })
dst := logBuffer.RegisterSubscriber("dst")
defer logBuffer.UnregisterSubscriber("dst")
woken := func() bool {
select {
case <-dst:
return true
case <-time.After(50 * time.Millisecond):
return false
}
}
notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src",
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/src"}})
if woken() {
t.Fatal("an event that stays in its directory must not wake other directories")
}
notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src",
EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "a"}, NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/dst"}})
if !woken() {
t.Fatal("a move must wake the destination directory's subscribers")
}
}
+1
View File
@@ -628,6 +628,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
glog.Errorf("failed to add data to log buffer for %s: %v", dir, err)
return err
}
notifyRenameTarget(ma.MetaLogBuffer, event)
// The event now sits at tsNs in the local stream; offset tracking
// still needs the peer's own position, so it is passed separately.
event.TsNs = tsNs
+48
View File
@@ -57,6 +57,54 @@ func ScanMetadataEventSkeleton(data []byte) (skeleton *SubscribeMetadataResponse
return skeleton, true
}
// ScanMetadataEventDirectories extracts an event's directory and its
// new_parent_path without allocating. ok=false means the data is malformed.
func ScanMetadataEventDirectories(data []byte) (directory, newParentPath []byte, ok bool) {
rest := data
for len(rest) > 0 {
num, typ, n := protowire.ConsumeTag(rest)
if n < 0 {
return nil, nil, false
}
rest = rest[n:]
if typ != protowire.BytesType || (num != 1 && num != 2) {
m := protowire.ConsumeFieldValue(num, typ, rest)
if m < 0 {
return nil, nil, false
}
rest = rest[m:]
continue
}
v, m := protowire.ConsumeBytes(rest)
if m < 0 {
return nil, nil, false
}
rest = rest[m:]
if num == 1 { // directory
directory = v
continue
}
// event_notification: repeated occurrences merge, so the last
// new_parent_path wins
for len(v) > 0 {
fnum, ftyp, fn := protowire.ConsumeTag(v)
if fn < 0 {
return nil, nil, false
}
v = v[fn:]
fm := protowire.ConsumeFieldValue(fnum, ftyp, v)
if fm < 0 {
return nil, nil, false
}
if fnum == 4 && ftyp == protowire.BytesType { // new_parent_path
newParentPath, _ = protowire.ConsumeBytes(v)
}
v = v[fm:]
}
}
return directory, newParentPath, true
}
func scanEventNotificationSkeleton(data []byte) (*EventNotification, bool) {
notification := &EventNotification{}
rest := data
@@ -110,3 +110,86 @@ func TestScanMetadataEventSkeletonEmptyPayload(t *testing.T) {
t.Fatalf("empty payload: %+v ok=%v", skeleton, ok)
}
}
func TestMetadataDirectoryMayMatchSubscriptionCoversMatches(t *testing.T) {
dirs := []string{"/", "/data", "/data/pvc-7", "/data/pvc-70", "/data/pvc-7/sub", "/data/pvc-1", "/other", "/data/./pvc-7", "/data//pvc-7", "/data/pvc-1/../pvc-7"}
names := []string{"pvc-7", "pvc-70", "a.log", "sub", ""}
filters := []struct {
prefix string
prefixes []string
dirs []string
}{
{prefix: "/data/pvc-7/"},
{prefix: "/data/pvc-7"},
{prefix: "/data/"},
{prefix: "/"},
{prefix: "/data/pvc-7/sub/deeper/"},
{prefix: "/x", prefixes: []string{"/other/", "/data/pvc-7/"}},
{prefix: "/x", dirs: []string{"/data/pvc-1"}},
}
for _, dir := range dirs {
for _, target := range append([]string{""}, dirs...) {
for _, name := range names {
event := &SubscribeMetadataResponse{Directory: dir, EventNotification: &EventNotification{
OldEntry: &Entry{Name: name}, NewEntry: &Entry{Name: name}, NewParentPath: target}}
for _, f := range filters {
if !MetadataEventMatchesSubscription(event, f.prefix, f.prefixes, f.dirs) {
continue
}
if !MetadataDirectoryMayMatchSubscription([]byte(dir), f.prefix, f.prefixes, f.dirs) &&
(target == "" || !MetadataDirectoryMayMatchSubscription([]byte(target), f.prefix, f.prefixes, f.dirs)) {
t.Errorf("event %q/%q -> %q matches %+v but neither directory may match", dir, name, target, f)
}
}
}
}
}
for _, c := range []struct {
dir, prefix string
want bool
}{
{"/data/pvc-1", "/data/pvc-7/", false},
{"/data/pvc-70", "/data/pvc-7/", false},
{"/other", "/data/", false},
{"/data/pvc-7", "/data/pvc-7/", true},
{"/data/pvc-7/sub", "/data/pvc-7/", true},
{"/data", "/data/pvc-7", true},
{"/", "/data/", true},
} {
if got := MetadataDirectoryMayMatchSubscription([]byte(c.dir), c.prefix, nil, nil); got != c.want {
t.Errorf("dir %q prefix %q: got %v, want %v", c.dir, c.prefix, got, c.want)
}
}
}
func TestScanMetadataEventDirectories(t *testing.T) {
for _, event := range []*SubscribeMetadataResponse{
{Directory: "/data/pvc-7", EventNotification: &EventNotification{NewEntry: bigEntry("a.log")}},
{Directory: "/data/pvc-7", EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("a.log"), NewParentPath: "/data/pvc-9"}},
{Directory: "/data/pvc-7"},
{},
} {
dir, target, ok := ScanMetadataEventDirectories(marshalEvent(t, event))
if !ok || string(dir) != event.Directory || string(target) != event.GetEventNotification().GetNewParentPath() {
t.Errorf("scan of %+v: dir %q target %q ok %v", event, dir, target, ok)
}
}
// repeated event_notification fields merge: the last new_parent_path wins
first := marshalEvent(t, &SubscribeMetadataResponse{Directory: "/a", EventNotification: &EventNotification{NewParentPath: "/b"}})
second := marshalEvent(t, &SubscribeMetadataResponse{EventNotification: &EventNotification{NewEntry: &Entry{Name: "x"}, NewParentPath: "/c"}})
merged := append(append([]byte{}, first...), second...)
full := &SubscribeMetadataResponse{}
if err := proto.Unmarshal(merged, full); err != nil {
t.Fatal(err)
}
dir, target, ok := ScanMetadataEventDirectories(merged)
if !ok || string(dir) != full.Directory || string(target) != full.EventNotification.NewParentPath {
t.Errorf("merged: dir %q target %q ok %v, full decode %q %q", dir, target, ok, full.Directory, full.EventNotification.NewParentPath)
}
if _, _, ok := ScanMetadataEventDirectories([]byte{0xff, 0xff, 0xff}); ok {
t.Error("malformed payload must not scan")
}
}
+38
View File
@@ -414,6 +414,44 @@ func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes
return strings.HasPrefix(fullPath, pathPrefix)
}
// MetadataDirectoryMayMatchSubscription reports whether an event in dir could
// match the subscription, before the event is decoded. It errs toward true.
func MetadataDirectoryMayMatchSubscription(dir []byte, pathPrefix string, pathPrefixes []string, directories []string) bool {
if pathPrefix == "" || len(dir) == 0 || directoryMayHavePrefix(dir, pathPrefix) {
return true
}
for _, p := range pathPrefixes {
if directoryMayHavePrefix(dir, p) {
return true
}
}
for _, d := range directories {
if string(dir) == d {
return true
}
}
// matching cleans the joined path, so only a clean dir can be ruled out
for i := 0; i+1 < len(dir); i++ {
if dir[i] == '/' && (dir[i+1] == '/' || dir[i+1] == '.') {
return true
}
}
return false
}
// directoryMayHavePrefix reports whether some path dir/<name> can start with
// prefix: dir+"/" and prefix agree on their common length.
func directoryMayHavePrefix(dir []byte, prefix string) bool {
n := min(len(dir), len(prefix))
if string(dir[:n]) != prefix[:n] {
return false
}
if len(prefix) <= len(dir) || dir[len(dir)-1] == '/' {
return true
}
return prefix[len(dir)] == '/'
}
func hasPrefixIn(text string, prefixes []string) bool {
for _, p := range prefixes {
if strings.HasPrefix(text, p) {
+26 -2
View File
@@ -683,6 +683,8 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
aggReaderName := fmt.Sprintf("aggMeta:%s:%d:%d", clientName, req.ClientId, req.ClientEpoch)
aggNotifyChan := fs.filer.MetaAggregator.MetaLogBuffer.RegisterSubscriber(aggNotifyName)
defer fs.filer.MetaAggregator.MetaLogBuffer.UnregisterSubscriber(aggNotifyName)
fs.filer.MetaAggregator.MetaLogBuffer.SetSubscriberWakeFilter(aggReaderName, subscriptionWakeFilter(req))
defer fs.filer.MetaAggregator.MetaLogBuffer.SetSubscriberWakeFilter(aggReaderName, nil)
gapStall := &gapStallReporter{scope: "aggregated", clientName: clientName, pathPrefix: req.PathPrefix}
defer gapStall.close()
@@ -1127,6 +1129,8 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
localReaderName := fmt.Sprintf("localMeta:%s:%d:%d", clientName, req.ClientId, req.ClientEpoch)
localFlushChan := fs.filer.LocalMetaLogBuffer.RegisterFlushSubscriber(localNotifyName)
defer fs.filer.LocalMetaLogBuffer.UnregisterFlushSubscriber(localNotifyName)
fs.filer.LocalMetaLogBuffer.SetSubscriberWakeFilter(localReaderName, subscriptionWakeFilter(req))
defer fs.filer.LocalMetaLogBuffer.SetSubscriberWakeFilter(localReaderName, nil)
gapStall := &gapStallReporter{scope: "local", clientName: clientName, pathPrefix: req.PathPrefix}
defer gapStall.close()
@@ -1305,11 +1309,32 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
}
// subscriptionWakeFilter lets a reader sleep through appends in directories
// the subscription cannot match; it reads them on its next poll instead.
func subscriptionWakeFilter(req *filer_pb.SubscribeMetadataRequest) func(key []byte) bool {
if req.PathPrefix == "" {
return nil
}
return func(key []byte) bool {
return filer_pb.MetadataDirectoryMayMatchSubscription(key, req.PathPrefix, req.PathPrefixes, req.Directories)
}
}
func eachLogEntryFn(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, sender metadataStreamSender, eachEventNotificationFn func(dirPath string, eventNotification *filer_pb.EventNotification, tsNs int64, mutationTsNs int64) error, filtered *int64) log_buffer.EachLogEntryFuncType {
// A shallow scan of the path fields skips unmarshaling chunk-heavy events
// this subscriber would filter out anyway; scan surprises fall back to the
// full decode. Only a delivery resets the shared unsynced-events counter.
prefilter := req.PathPrefix != "" || len(req.PathPrefixes) > 0 || len(req.Directories) > 0
mayMatch := subscriptionWakeFilter(req)
skip := func(data []byte) bool {
if mayMatch != nil {
if dir, target, ok := filer_pb.ScanMetadataEventDirectories(data); ok && !mayMatch(dir) && (len(target) == 0 || !mayMatch(target)) {
return true
}
}
skeleton, ok := filer_pb.ScanMetadataEventSkeleton(data)
return ok && !filer_pb.MetadataEventMatchesSubscription(skeleton, req.PathPrefix, req.PathPrefixes, req.Directories)
}
return func(logEntry *filer_pb.LogEntry) (bool, error) {
// A cancelled context ends the pass here: neither loop checks it
// per entry, and the send path alone cannot cover filtered entries.
@@ -1317,8 +1342,7 @@ func eachLogEntryFn(ctx context.Context, req *filer_pb.SubscribeMetadataRequest,
return true, nil
}
if prefilter {
if skeleton, ok := filer_pb.ScanMetadataEventSkeleton(logEntry.Data); ok &&
!filer_pb.MetadataEventMatchesSubscription(skeleton, req.PathPrefix, req.PathPrefixes, req.Directories) {
if skip(logEntry.Data) {
*filtered++
if *filtered > MaxUnsyncedEvents {
if err := sender.Send(&filer_pb.SubscribeMetadataResponse{
@@ -158,3 +158,33 @@ func TestEachLogEntryFnFilteredHeartbeatUsesStreamPosition(t *testing.T) {
t.Fatalf("heartbeat ts = %+v, want stream position 101", sender.sent)
}
}
func TestEachLogEntryFnPrefilterKeepsMovesIntoPrefix(t *testing.T) {
req := &filer_pb.SubscribeMetadataRequest{PathPrefix: "/data/pvc-7/"}
var delivered []string
var unsyncedEvents int64
fn := eachLogEntryFn(context.Background(), req, &recordingSender{}, func(dirPath string, eventNotification *filer_pb.EventNotification, tsNs int64, mutationTsNs int64) error {
delivered = append(delivered, eventNotification.NewParentPath)
return nil
}, &unsyncedEvents)
for i, target := range []string{"/data/pvc-7", "/data/pvc-1/sub"} {
data, err := proto.Marshal(&filer_pb.SubscribeMetadataResponse{
Directory: "/data/pvc-1",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "a.log"},
NewEntry: &filer_pb.Entry{Name: "a.log"},
NewParentPath: target,
},
})
if err != nil {
t.Fatal(err)
}
if _, err := fn(&filer_pb.LogEntry{TsNs: int64(i + 1), Data: data}); err != nil {
t.Fatal(err)
}
}
if len(delivered) != 1 || delivered[0] != "/data/pvc-7" {
t.Fatalf("only the move into the prefix must reach the notification fn, got %v", delivered)
}
}
+42 -16
View File
@@ -194,8 +194,11 @@ type LogBuffer struct {
subscribers map[string]*subscription // subscriberID -> shared notification channel
// Notified only when a flush lands, for readers that cannot act on an append
flushSubscribers map[string]*subscription
isStopping *atomic.Bool
shutdownCh chan struct{} // closed by ShutdownLogBuffer to wake blocked subscribers
// wakeFilters skip a subscriber's append wake-up when the entry's partition
// key cannot interest it; it still reads the entry on its next poll.
wakeFilters map[string]func(key []byte) bool
isStopping *atomic.Bool
shutdownCh chan struct{} // closed by ShutdownLogBuffer to wake blocked subscribers
// flushDroppedBytes counts bytes the flushFn gave up on during shutdown.
// loopFlush reads it after each window so an abandoned flush does not
// advance the flushed watermark or wake subscribers as if it had landed.
@@ -232,6 +235,7 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn LogFlushFunc
notifyFn: notifyFn,
subscribers: make(map[string]*subscription),
flushSubscribers: make(map[string]*subscription),
wakeFilters: make(map[string]func(key []byte) bool),
flushChan: make(chan *dataToFlush, flushQueueDepth),
isStopping: new(atomic.Bool),
shutdownCh: make(chan struct{}),
@@ -259,6 +263,7 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn LogFlushFunc
type subscription struct {
notifyChan chan struct{}
refCount int
wants func(key []byte) bool
}
func registerSubscription(subscriptions map[string]*subscription, subscriberID string) chan struct{} {
@@ -292,7 +297,9 @@ func (logBuffer *LogBuffer) RegisterSubscriber(subscriberID string) chan struct{
logBuffer.subscribersMu.Lock()
defer logBuffer.subscribersMu.Unlock()
return registerSubscription(logBuffer.subscribers, subscriberID)
notifyChan := registerSubscription(logBuffer.subscribers, subscriberID)
logBuffer.subscribers[subscriberID].wants = logBuffer.wakeFilters[subscriberID]
return notifyChan
}
// UnregisterSubscriber removes a subscriber and closes its notification channel
@@ -374,23 +381,42 @@ func (logBuffer *LogBuffer) IsOffsetInMemory(offset int64) bool {
return false
}
// notifySubscribers sends notifications to all registered subscribers
// Non-blocking: uses select with default to avoid blocking on full channels
func (logBuffer *LogBuffer) notifySubscribers() {
// SetSubscriberWakeFilter limits append wake-ups of the subscriber registered
// as subscriberID to entries whose partition key satisfies wants; a nil wants
// removes the filter. Flushes and keyless appends still wake every subscriber.
func (logBuffer *LogBuffer) SetSubscriberWakeFilter(subscriberID string, wants func(key []byte) bool) {
logBuffer.subscribersMu.Lock()
defer logBuffer.subscribersMu.Unlock()
if wants == nil {
delete(logBuffer.wakeFilters, subscriberID)
} else {
logBuffer.wakeFilters[subscriberID] = wants
}
if sub, ok := logBuffer.subscribers[subscriberID]; ok {
sub.wants = wants
}
}
// NotifySubscribers wakes the subscribers interested in key, for entries that
// also concern a key other than the partition key they were appended under.
func (logBuffer *LogBuffer) NotifySubscribers(key []byte) {
logBuffer.notifySubscribers(key)
}
// notifySubscribers wakes the subscribers interested in key, or all of them
// for an empty key. Non-blocking: a full channel already holds a wake-up.
func (logBuffer *LogBuffer) notifySubscribers(key []byte) {
logBuffer.subscribersMu.RLock()
defer logBuffer.subscribersMu.RUnlock()
if len(logBuffer.subscribers) == 0 {
return // No subscribers, skip notification
}
for _, sub := range logBuffer.subscribers {
if len(key) > 0 && sub.wants != nil && !sub.wants(key) {
continue
}
select {
case sub.notifyChan <- struct{}{}:
// Notification sent successfully
default:
// Channel full - subscriber hasn't consumed previous notification yet
// This is OK because one notification is sufficient to wake the subscriber
}
}
}
@@ -468,7 +494,7 @@ func (logBuffer *LogBuffer) AddLogEntryToBuffer(logEntry *filer_pb.LogEntry) err
logBuffer.notifyFn()
}
// Notify all registered subscribers instantly (<1ms latency)
logBuffer.notifySubscribers()
logBuffer.notifySubscribers(logEntry.Key)
}
}()
@@ -607,7 +633,7 @@ func (logBuffer *LogBuffer) AddDataToBufferAndGetTs(partitionKey, data []byte, p
logBuffer.notifyFn()
}
// Notify all registered subscribers instantly (<1ms latency)
logBuffer.notifySubscribers()
logBuffer.notifySubscribers(partitionKey)
}
}()
@@ -823,7 +849,7 @@ func (logBuffer *LogBuffer) loopFlush() {
if logBuffer.notifyFn != nil {
logBuffer.notifyFn()
}
logBuffer.notifySubscribers()
logBuffer.notifySubscribers(nil)
logBuffer.notifyFlushSubscribers()
// Signal completion if there's a callback channel
@@ -0,0 +1,97 @@
package log_buffer
import (
"bytes"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func woken(ch <-chan struct{}) bool {
select {
case <-ch:
return true
case <-time.After(50 * time.Millisecond):
return false
}
}
func TestSubscriberWakeFilter(t *testing.T) {
flushFn := func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}
logBuffer := NewLogBuffer("test", time.Minute, flushFn, nil, nil)
defer logBuffer.ShutdownLogBuffer()
wantsA := func(key []byte) bool { return bytes.Equal(key, []byte("a")) }
// set before registration, the way a reader is set up ahead of its loop
logBuffer.SetSubscriberWakeFilter("filtered", wantsA)
filtered := logBuffer.RegisterSubscriber("filtered")
defer logBuffer.UnregisterSubscriber("filtered")
all := logBuffer.RegisterSubscriber("all")
defer logBuffer.UnregisterSubscriber("all")
late := logBuffer.RegisterSubscriber("late")
defer logBuffer.UnregisterSubscriber("late")
logBuffer.SetSubscriberWakeFilter("late", wantsA)
if err := logBuffer.AddDataToBuffer([]byte("b"), []byte("v"), 0); err != nil {
t.Fatal(err)
}
if woken(filtered) || woken(late) {
t.Fatal("an append under another key must not wake filtered subscribers")
}
if !woken(all) {
t.Fatal("an unfiltered subscriber must wake on every append")
}
if err := logBuffer.AddDataToBuffer([]byte("a"), []byte("v"), 0); err != nil {
t.Fatal(err)
}
if !woken(filtered) || !woken(late) || !woken(all) {
t.Fatal("an append under a wanted key must wake every subscriber")
}
logBuffer.NotifySubscribers([]byte("a"))
if !woken(filtered) {
t.Fatal("NotifySubscribers must wake subscribers that want the key")
}
logBuffer.ForceFlush()
if !woken(filtered) || !woken(late) {
t.Fatal("a flush must wake filtered subscribers")
}
logBuffer.SetSubscriberWakeFilter("late", nil)
if err := logBuffer.AddDataToBuffer([]byte("b"), []byte("v"), 0); err != nil {
t.Fatal(err)
}
if !woken(late) {
t.Fatal("clearing the filter must restore wake-ups")
}
}
func TestSubscriberWakeFilterStillDelivers(t *testing.T) {
flushFn := func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}
logBuffer := NewLogBuffer("test", time.Minute, flushFn, nil, nil)
defer logBuffer.ShutdownLogBuffer()
logBuffer.SetSubscriberWakeFilter("reader", func([]byte) bool { return false })
start := NewMessagePosition(time.Now().UnixNano(), -2)
got := make(chan []byte, 1)
go logBuffer.LoopProcessLogData("reader", start, 0, func() bool { return true }, func(logEntry *filer_pb.LogEntry) (bool, error) {
got <- bytes.Clone(logEntry.Data)
return true, nil
})
time.Sleep(50 * time.Millisecond)
if err := logBuffer.AddDataToBuffer([]byte("unwanted"), []byte("v"), 0); err != nil {
t.Fatal(err)
}
select {
case data := <-got:
if string(data) != "v" {
t.Fatalf("got %q", data)
}
case <-time.After(2 * notificationHealthCheckInterval):
t.Fatal("a reader that sleeps through an append must still read it on its next poll")
}
}