diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index e71a8b90a1..abb49e046b 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -288,6 +288,14 @@ func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFrom } } +// notifyRenameTarget wakes the subscribers of a move's destination directory, +// which the partition key (the source directory) does not cover. +func notifyRenameTarget(logBuffer *log_buffer.LogBuffer, event *filer_pb.SubscribeMetadataResponse) { + if target := filer_pb.MetadataEventTargetDirectory(event); target != event.Directory { + logBuffer.NotifySubscribers([]byte(target)) + } +} + func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) { data, err := proto.Marshal(event) if err != nil { @@ -300,6 +308,7 @@ func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetad glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err) return } + notifyRenameTarget(f.LocalMetaLogBuffer, event) // The in-memory copy now carries the position the stream assigns it, the // same value subscribers will see delivered. event.TsNs = tsNs diff --git a/weed/filer/filer_notify_rename_test.go b/weed/filer/filer_notify_rename_test.go new file mode 100644 index 0000000000..f7a0e9f773 --- /dev/null +++ b/weed/filer/filer_notify_rename_test.go @@ -0,0 +1,40 @@ +package filer + +import ( + "bytes" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util/log_buffer" +) + +func TestNotifyRenameTargetWakesDestinationSubscribers(t *testing.T) { + logBuffer := log_buffer.NewLogBuffer("test", time.Minute, func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil) + defer logBuffer.ShutdownLogBuffer() + + logBuffer.SetSubscriberWakeFilter("dst", func(key []byte) bool { return bytes.Equal(key, []byte("/dst")) }) + dst := logBuffer.RegisterSubscriber("dst") + defer logBuffer.UnregisterSubscriber("dst") + + woken := func() bool { + select { + case <-dst: + return true + case <-time.After(50 * time.Millisecond): + return false + } + } + + notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src", + EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/src"}}) + if woken() { + t.Fatal("an event that stays in its directory must not wake other directories") + } + + notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src", + EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "a"}, NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/dst"}}) + if !woken() { + t.Fatal("a move must wake the destination directory's subscribers") + } +} diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index 367a74f678..ac736946ed 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -628,6 +628,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, glog.Errorf("failed to add data to log buffer for %s: %v", dir, err) return err } + notifyRenameTarget(ma.MetaLogBuffer, event) // The event now sits at tsNs in the local stream; offset tracking // still needs the peer's own position, so it is passed separately. event.TsNs = tsNs diff --git a/weed/pb/filer_pb/filer_pb_event_scan.go b/weed/pb/filer_pb/filer_pb_event_scan.go index b99a73e872..81aef43974 100644 --- a/weed/pb/filer_pb/filer_pb_event_scan.go +++ b/weed/pb/filer_pb/filer_pb_event_scan.go @@ -57,6 +57,54 @@ func ScanMetadataEventSkeleton(data []byte) (skeleton *SubscribeMetadataResponse return skeleton, true } +// ScanMetadataEventDirectories extracts an event's directory and its +// new_parent_path without allocating. ok=false means the data is malformed. +func ScanMetadataEventDirectories(data []byte) (directory, newParentPath []byte, ok bool) { + rest := data + for len(rest) > 0 { + num, typ, n := protowire.ConsumeTag(rest) + if n < 0 { + return nil, nil, false + } + rest = rest[n:] + if typ != protowire.BytesType || (num != 1 && num != 2) { + m := protowire.ConsumeFieldValue(num, typ, rest) + if m < 0 { + return nil, nil, false + } + rest = rest[m:] + continue + } + v, m := protowire.ConsumeBytes(rest) + if m < 0 { + return nil, nil, false + } + rest = rest[m:] + if num == 1 { // directory + directory = v + continue + } + // event_notification: repeated occurrences merge, so the last + // new_parent_path wins + for len(v) > 0 { + fnum, ftyp, fn := protowire.ConsumeTag(v) + if fn < 0 { + return nil, nil, false + } + v = v[fn:] + fm := protowire.ConsumeFieldValue(fnum, ftyp, v) + if fm < 0 { + return nil, nil, false + } + if fnum == 4 && ftyp == protowire.BytesType { // new_parent_path + newParentPath, _ = protowire.ConsumeBytes(v) + } + v = v[fm:] + } + } + return directory, newParentPath, true +} + func scanEventNotificationSkeleton(data []byte) (*EventNotification, bool) { notification := &EventNotification{} rest := data diff --git a/weed/pb/filer_pb/filer_pb_event_scan_test.go b/weed/pb/filer_pb/filer_pb_event_scan_test.go index aa8498ea47..abe1e99bde 100644 --- a/weed/pb/filer_pb/filer_pb_event_scan_test.go +++ b/weed/pb/filer_pb/filer_pb_event_scan_test.go @@ -110,3 +110,86 @@ func TestScanMetadataEventSkeletonEmptyPayload(t *testing.T) { t.Fatalf("empty payload: %+v ok=%v", skeleton, ok) } } + +func TestMetadataDirectoryMayMatchSubscriptionCoversMatches(t *testing.T) { + dirs := []string{"/", "/data", "/data/pvc-7", "/data/pvc-70", "/data/pvc-7/sub", "/data/pvc-1", "/other", "/data/./pvc-7", "/data//pvc-7", "/data/pvc-1/../pvc-7"} + names := []string{"pvc-7", "pvc-70", "a.log", "sub", ""} + filters := []struct { + prefix string + prefixes []string + dirs []string + }{ + {prefix: "/data/pvc-7/"}, + {prefix: "/data/pvc-7"}, + {prefix: "/data/"}, + {prefix: "/"}, + {prefix: "/data/pvc-7/sub/deeper/"}, + {prefix: "/x", prefixes: []string{"/other/", "/data/pvc-7/"}}, + {prefix: "/x", dirs: []string{"/data/pvc-1"}}, + } + for _, dir := range dirs { + for _, target := range append([]string{""}, dirs...) { + for _, name := range names { + event := &SubscribeMetadataResponse{Directory: dir, EventNotification: &EventNotification{ + OldEntry: &Entry{Name: name}, NewEntry: &Entry{Name: name}, NewParentPath: target}} + for _, f := range filters { + if !MetadataEventMatchesSubscription(event, f.prefix, f.prefixes, f.dirs) { + continue + } + if !MetadataDirectoryMayMatchSubscription([]byte(dir), f.prefix, f.prefixes, f.dirs) && + (target == "" || !MetadataDirectoryMayMatchSubscription([]byte(target), f.prefix, f.prefixes, f.dirs)) { + t.Errorf("event %q/%q -> %q matches %+v but neither directory may match", dir, name, target, f) + } + } + } + } + } + + for _, c := range []struct { + dir, prefix string + want bool + }{ + {"/data/pvc-1", "/data/pvc-7/", false}, + {"/data/pvc-70", "/data/pvc-7/", false}, + {"/other", "/data/", false}, + {"/data/pvc-7", "/data/pvc-7/", true}, + {"/data/pvc-7/sub", "/data/pvc-7/", true}, + {"/data", "/data/pvc-7", true}, + {"/", "/data/", true}, + } { + if got := MetadataDirectoryMayMatchSubscription([]byte(c.dir), c.prefix, nil, nil); got != c.want { + t.Errorf("dir %q prefix %q: got %v, want %v", c.dir, c.prefix, got, c.want) + } + } +} + +func TestScanMetadataEventDirectories(t *testing.T) { + for _, event := range []*SubscribeMetadataResponse{ + {Directory: "/data/pvc-7", EventNotification: &EventNotification{NewEntry: bigEntry("a.log")}}, + {Directory: "/data/pvc-7", EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("a.log"), NewParentPath: "/data/pvc-9"}}, + {Directory: "/data/pvc-7"}, + {}, + } { + dir, target, ok := ScanMetadataEventDirectories(marshalEvent(t, event)) + if !ok || string(dir) != event.Directory || string(target) != event.GetEventNotification().GetNewParentPath() { + t.Errorf("scan of %+v: dir %q target %q ok %v", event, dir, target, ok) + } + } + + // repeated event_notification fields merge: the last new_parent_path wins + first := marshalEvent(t, &SubscribeMetadataResponse{Directory: "/a", EventNotification: &EventNotification{NewParentPath: "/b"}}) + second := marshalEvent(t, &SubscribeMetadataResponse{EventNotification: &EventNotification{NewEntry: &Entry{Name: "x"}, NewParentPath: "/c"}}) + merged := append(append([]byte{}, first...), second...) + full := &SubscribeMetadataResponse{} + if err := proto.Unmarshal(merged, full); err != nil { + t.Fatal(err) + } + dir, target, ok := ScanMetadataEventDirectories(merged) + if !ok || string(dir) != full.Directory || string(target) != full.EventNotification.NewParentPath { + t.Errorf("merged: dir %q target %q ok %v, full decode %q %q", dir, target, ok, full.Directory, full.EventNotification.NewParentPath) + } + + if _, _, ok := ScanMetadataEventDirectories([]byte{0xff, 0xff, 0xff}); ok { + t.Error("malformed payload must not scan") + } +} diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index 21d344f6ac..e5ff2ce4ff 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -414,6 +414,44 @@ func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes return strings.HasPrefix(fullPath, pathPrefix) } +// MetadataDirectoryMayMatchSubscription reports whether an event in dir could +// match the subscription, before the event is decoded. It errs toward true. +func MetadataDirectoryMayMatchSubscription(dir []byte, pathPrefix string, pathPrefixes []string, directories []string) bool { + if pathPrefix == "" || len(dir) == 0 || directoryMayHavePrefix(dir, pathPrefix) { + return true + } + for _, p := range pathPrefixes { + if directoryMayHavePrefix(dir, p) { + return true + } + } + for _, d := range directories { + if string(dir) == d { + return true + } + } + // matching cleans the joined path, so only a clean dir can be ruled out + for i := 0; i+1 < len(dir); i++ { + if dir[i] == '/' && (dir[i+1] == '/' || dir[i+1] == '.') { + return true + } + } + return false +} + +// directoryMayHavePrefix reports whether some path dir/ can start with +// prefix: dir+"/" and prefix agree on their common length. +func directoryMayHavePrefix(dir []byte, prefix string) bool { + n := min(len(dir), len(prefix)) + if string(dir[:n]) != prefix[:n] { + return false + } + if len(prefix) <= len(dir) || dir[len(dir)-1] == '/' { + return true + } + return prefix[len(dir)] == '/' +} + func hasPrefixIn(text string, prefixes []string) bool { for _, p := range prefixes { if strings.HasPrefix(text, p) { diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index 66af39618a..d3ba194aae 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -683,6 +683,8 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, aggReaderName := fmt.Sprintf("aggMeta:%s:%d:%d", clientName, req.ClientId, req.ClientEpoch) aggNotifyChan := fs.filer.MetaAggregator.MetaLogBuffer.RegisterSubscriber(aggNotifyName) defer fs.filer.MetaAggregator.MetaLogBuffer.UnregisterSubscriber(aggNotifyName) + fs.filer.MetaAggregator.MetaLogBuffer.SetSubscriberWakeFilter(aggReaderName, subscriptionWakeFilter(req)) + defer fs.filer.MetaAggregator.MetaLogBuffer.SetSubscriberWakeFilter(aggReaderName, nil) gapStall := &gapStallReporter{scope: "aggregated", clientName: clientName, pathPrefix: req.PathPrefix} defer gapStall.close() @@ -1127,6 +1129,8 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq localReaderName := fmt.Sprintf("localMeta:%s:%d:%d", clientName, req.ClientId, req.ClientEpoch) localFlushChan := fs.filer.LocalMetaLogBuffer.RegisterFlushSubscriber(localNotifyName) defer fs.filer.LocalMetaLogBuffer.UnregisterFlushSubscriber(localNotifyName) + fs.filer.LocalMetaLogBuffer.SetSubscriberWakeFilter(localReaderName, subscriptionWakeFilter(req)) + defer fs.filer.LocalMetaLogBuffer.SetSubscriberWakeFilter(localReaderName, nil) gapStall := &gapStallReporter{scope: "local", clientName: clientName, pathPrefix: req.PathPrefix} defer gapStall.close() @@ -1305,11 +1309,32 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq } +// subscriptionWakeFilter lets a reader sleep through appends in directories +// the subscription cannot match; it reads them on its next poll instead. +func subscriptionWakeFilter(req *filer_pb.SubscribeMetadataRequest) func(key []byte) bool { + if req.PathPrefix == "" { + return nil + } + return func(key []byte) bool { + return filer_pb.MetadataDirectoryMayMatchSubscription(key, req.PathPrefix, req.PathPrefixes, req.Directories) + } +} + func eachLogEntryFn(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, sender metadataStreamSender, eachEventNotificationFn func(dirPath string, eventNotification *filer_pb.EventNotification, tsNs int64, mutationTsNs int64) error, filtered *int64) log_buffer.EachLogEntryFuncType { // A shallow scan of the path fields skips unmarshaling chunk-heavy events // this subscriber would filter out anyway; scan surprises fall back to the // full decode. Only a delivery resets the shared unsynced-events counter. prefilter := req.PathPrefix != "" || len(req.PathPrefixes) > 0 || len(req.Directories) > 0 + mayMatch := subscriptionWakeFilter(req) + skip := func(data []byte) bool { + if mayMatch != nil { + if dir, target, ok := filer_pb.ScanMetadataEventDirectories(data); ok && !mayMatch(dir) && (len(target) == 0 || !mayMatch(target)) { + return true + } + } + skeleton, ok := filer_pb.ScanMetadataEventSkeleton(data) + return ok && !filer_pb.MetadataEventMatchesSubscription(skeleton, req.PathPrefix, req.PathPrefixes, req.Directories) + } return func(logEntry *filer_pb.LogEntry) (bool, error) { // A cancelled context ends the pass here: neither loop checks it // per entry, and the send path alone cannot cover filtered entries. @@ -1317,8 +1342,7 @@ func eachLogEntryFn(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, return true, nil } if prefilter { - if skeleton, ok := filer_pb.ScanMetadataEventSkeleton(logEntry.Data); ok && - !filer_pb.MetadataEventMatchesSubscription(skeleton, req.PathPrefix, req.PathPrefixes, req.Directories) { + if skip(logEntry.Data) { *filtered++ if *filtered > MaxUnsyncedEvents { if err := sender.Send(&filer_pb.SubscribeMetadataResponse{ diff --git a/weed/server/filer_grpc_server_sub_meta_prefilter_test.go b/weed/server/filer_grpc_server_sub_meta_prefilter_test.go index b435a9b142..798e58b39d 100644 --- a/weed/server/filer_grpc_server_sub_meta_prefilter_test.go +++ b/weed/server/filer_grpc_server_sub_meta_prefilter_test.go @@ -158,3 +158,33 @@ func TestEachLogEntryFnFilteredHeartbeatUsesStreamPosition(t *testing.T) { t.Fatalf("heartbeat ts = %+v, want stream position 101", sender.sent) } } + +func TestEachLogEntryFnPrefilterKeepsMovesIntoPrefix(t *testing.T) { + req := &filer_pb.SubscribeMetadataRequest{PathPrefix: "/data/pvc-7/"} + var delivered []string + var unsyncedEvents int64 + fn := eachLogEntryFn(context.Background(), req, &recordingSender{}, func(dirPath string, eventNotification *filer_pb.EventNotification, tsNs int64, mutationTsNs int64) error { + delivered = append(delivered, eventNotification.NewParentPath) + return nil + }, &unsyncedEvents) + + for i, target := range []string{"/data/pvc-7", "/data/pvc-1/sub"} { + data, err := proto.Marshal(&filer_pb.SubscribeMetadataResponse{ + Directory: "/data/pvc-1", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "a.log"}, + NewEntry: &filer_pb.Entry{Name: "a.log"}, + NewParentPath: target, + }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := fn(&filer_pb.LogEntry{TsNs: int64(i + 1), Data: data}); err != nil { + t.Fatal(err) + } + } + if len(delivered) != 1 || delivered[0] != "/data/pvc-7" { + t.Fatalf("only the move into the prefix must reach the notification fn, got %v", delivered) + } +} diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go index 008731d9f2..e0a6c0e649 100644 --- a/weed/util/log_buffer/log_buffer.go +++ b/weed/util/log_buffer/log_buffer.go @@ -194,8 +194,11 @@ type LogBuffer struct { subscribers map[string]*subscription // subscriberID -> shared notification channel // Notified only when a flush lands, for readers that cannot act on an append flushSubscribers map[string]*subscription - isStopping *atomic.Bool - shutdownCh chan struct{} // closed by ShutdownLogBuffer to wake blocked subscribers + // wakeFilters skip a subscriber's append wake-up when the entry's partition + // key cannot interest it; it still reads the entry on its next poll. + wakeFilters map[string]func(key []byte) bool + isStopping *atomic.Bool + shutdownCh chan struct{} // closed by ShutdownLogBuffer to wake blocked subscribers // flushDroppedBytes counts bytes the flushFn gave up on during shutdown. // loopFlush reads it after each window so an abandoned flush does not // advance the flushed watermark or wake subscribers as if it had landed. @@ -232,6 +235,7 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn LogFlushFunc notifyFn: notifyFn, subscribers: make(map[string]*subscription), flushSubscribers: make(map[string]*subscription), + wakeFilters: make(map[string]func(key []byte) bool), flushChan: make(chan *dataToFlush, flushQueueDepth), isStopping: new(atomic.Bool), shutdownCh: make(chan struct{}), @@ -259,6 +263,7 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn LogFlushFunc type subscription struct { notifyChan chan struct{} refCount int + wants func(key []byte) bool } func registerSubscription(subscriptions map[string]*subscription, subscriberID string) chan struct{} { @@ -292,7 +297,9 @@ func (logBuffer *LogBuffer) RegisterSubscriber(subscriberID string) chan struct{ logBuffer.subscribersMu.Lock() defer logBuffer.subscribersMu.Unlock() - return registerSubscription(logBuffer.subscribers, subscriberID) + notifyChan := registerSubscription(logBuffer.subscribers, subscriberID) + logBuffer.subscribers[subscriberID].wants = logBuffer.wakeFilters[subscriberID] + return notifyChan } // UnregisterSubscriber removes a subscriber and closes its notification channel @@ -374,23 +381,42 @@ func (logBuffer *LogBuffer) IsOffsetInMemory(offset int64) bool { return false } -// notifySubscribers sends notifications to all registered subscribers -// Non-blocking: uses select with default to avoid blocking on full channels -func (logBuffer *LogBuffer) notifySubscribers() { +// SetSubscriberWakeFilter limits append wake-ups of the subscriber registered +// as subscriberID to entries whose partition key satisfies wants; a nil wants +// removes the filter. Flushes and keyless appends still wake every subscriber. +func (logBuffer *LogBuffer) SetSubscriberWakeFilter(subscriberID string, wants func(key []byte) bool) { + logBuffer.subscribersMu.Lock() + defer logBuffer.subscribersMu.Unlock() + + if wants == nil { + delete(logBuffer.wakeFilters, subscriberID) + } else { + logBuffer.wakeFilters[subscriberID] = wants + } + if sub, ok := logBuffer.subscribers[subscriberID]; ok { + sub.wants = wants + } +} + +// NotifySubscribers wakes the subscribers interested in key, for entries that +// also concern a key other than the partition key they were appended under. +func (logBuffer *LogBuffer) NotifySubscribers(key []byte) { + logBuffer.notifySubscribers(key) +} + +// notifySubscribers wakes the subscribers interested in key, or all of them +// for an empty key. Non-blocking: a full channel already holds a wake-up. +func (logBuffer *LogBuffer) notifySubscribers(key []byte) { logBuffer.subscribersMu.RLock() defer logBuffer.subscribersMu.RUnlock() - if len(logBuffer.subscribers) == 0 { - return // No subscribers, skip notification - } - for _, sub := range logBuffer.subscribers { + if len(key) > 0 && sub.wants != nil && !sub.wants(key) { + continue + } select { case sub.notifyChan <- struct{}{}: - // Notification sent successfully default: - // Channel full - subscriber hasn't consumed previous notification yet - // This is OK because one notification is sufficient to wake the subscriber } } } @@ -468,7 +494,7 @@ func (logBuffer *LogBuffer) AddLogEntryToBuffer(logEntry *filer_pb.LogEntry) err logBuffer.notifyFn() } // Notify all registered subscribers instantly (<1ms latency) - logBuffer.notifySubscribers() + logBuffer.notifySubscribers(logEntry.Key) } }() @@ -607,7 +633,7 @@ func (logBuffer *LogBuffer) AddDataToBufferAndGetTs(partitionKey, data []byte, p logBuffer.notifyFn() } // Notify all registered subscribers instantly (<1ms latency) - logBuffer.notifySubscribers() + logBuffer.notifySubscribers(partitionKey) } }() @@ -823,7 +849,7 @@ func (logBuffer *LogBuffer) loopFlush() { if logBuffer.notifyFn != nil { logBuffer.notifyFn() } - logBuffer.notifySubscribers() + logBuffer.notifySubscribers(nil) logBuffer.notifyFlushSubscribers() // Signal completion if there's a callback channel diff --git a/weed/util/log_buffer/subscriber_wake_filter_test.go b/weed/util/log_buffer/subscriber_wake_filter_test.go new file mode 100644 index 0000000000..3f437e6a95 --- /dev/null +++ b/weed/util/log_buffer/subscriber_wake_filter_test.go @@ -0,0 +1,97 @@ +package log_buffer + +import ( + "bytes" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +func woken(ch <-chan struct{}) bool { + select { + case <-ch: + return true + case <-time.After(50 * time.Millisecond): + return false + } +} + +func TestSubscriberWakeFilter(t *testing.T) { + flushFn := func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {} + logBuffer := NewLogBuffer("test", time.Minute, flushFn, nil, nil) + defer logBuffer.ShutdownLogBuffer() + + wantsA := func(key []byte) bool { return bytes.Equal(key, []byte("a")) } + // set before registration, the way a reader is set up ahead of its loop + logBuffer.SetSubscriberWakeFilter("filtered", wantsA) + filtered := logBuffer.RegisterSubscriber("filtered") + defer logBuffer.UnregisterSubscriber("filtered") + all := logBuffer.RegisterSubscriber("all") + defer logBuffer.UnregisterSubscriber("all") + late := logBuffer.RegisterSubscriber("late") + defer logBuffer.UnregisterSubscriber("late") + logBuffer.SetSubscriberWakeFilter("late", wantsA) + + if err := logBuffer.AddDataToBuffer([]byte("b"), []byte("v"), 0); err != nil { + t.Fatal(err) + } + if woken(filtered) || woken(late) { + t.Fatal("an append under another key must not wake filtered subscribers") + } + if !woken(all) { + t.Fatal("an unfiltered subscriber must wake on every append") + } + + if err := logBuffer.AddDataToBuffer([]byte("a"), []byte("v"), 0); err != nil { + t.Fatal(err) + } + if !woken(filtered) || !woken(late) || !woken(all) { + t.Fatal("an append under a wanted key must wake every subscriber") + } + + logBuffer.NotifySubscribers([]byte("a")) + if !woken(filtered) { + t.Fatal("NotifySubscribers must wake subscribers that want the key") + } + + logBuffer.ForceFlush() + if !woken(filtered) || !woken(late) { + t.Fatal("a flush must wake filtered subscribers") + } + + logBuffer.SetSubscriberWakeFilter("late", nil) + if err := logBuffer.AddDataToBuffer([]byte("b"), []byte("v"), 0); err != nil { + t.Fatal(err) + } + if !woken(late) { + t.Fatal("clearing the filter must restore wake-ups") + } +} + +func TestSubscriberWakeFilterStillDelivers(t *testing.T) { + flushFn := func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {} + logBuffer := NewLogBuffer("test", time.Minute, flushFn, nil, nil) + defer logBuffer.ShutdownLogBuffer() + + logBuffer.SetSubscriberWakeFilter("reader", func([]byte) bool { return false }) + start := NewMessagePosition(time.Now().UnixNano(), -2) + got := make(chan []byte, 1) + go logBuffer.LoopProcessLogData("reader", start, 0, func() bool { return true }, func(logEntry *filer_pb.LogEntry) (bool, error) { + got <- bytes.Clone(logEntry.Data) + return true, nil + }) + + time.Sleep(50 * time.Millisecond) + if err := logBuffer.AddDataToBuffer([]byte("unwanted"), []byte("v"), 0); err != nil { + t.Fatal(err) + } + select { + case data := <-got: + if string(data) != "v" { + t.Fatalf("got %q", data) + } + case <-time.After(2 * notificationHealthCheckInterval): + t.Fatal("a reader that sleeps through an append must still read it on its next poll") + } +}