Files
seaweedfs/weed/filer/filer_notify.go
T
Chris LuandClaude Opus 5.5 8e15110b4f filer: wake metadata subscribers only for directories they can match (#12669)
* log_buffer: let a subscriber skip wake-ups for unwanted partition keys

Every append woke every registered subscriber. SetSubscriberWakeFilter lets
a reader sleep through appends whose partition key it would filter out; it
still reads them on its next poll. Flushes and keyless appends wake everyone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer_pb: decide subscription matches from an event's directories

MetadataDirectoryMayMatchSubscription rules out events whose directory no
child path of can match the subscription, and ScanMetadataEventDirectories
reads an event's directory and new parent path without allocating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: wake subscribers of a move's destination directory

Appends are keyed by the source directory, so a filtered subscriber of the
destination would otherwise only see the move on its next poll.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: wake metadata subscribers only for directories they can match

Every metadata event woke and was scanned by every subscriber, so filer CPU
grew with the number of subscribers watching other paths. Readers now
register a directory wake filter, and the per-entry prefilter rules entries
out from their directories before the skeleton scan.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test that moves into a subscribed prefix pass the prefilter

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 20:53:41 -08:00

530 lines
17 KiB
Go

package filer
import (
"context"
"errors"
"fmt"
"io"
nethttp "net/http"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"google.golang.org/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
f.notifyUpdateEvent(ctx, oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures)
}
func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
if metadataEventsSuppressed(ctx) {
// The entry's stamp stays in flight for the covering event the
// caller emits itself (the rename handler does).
return nil
}
var fullpath string
if oldEntry != nil {
fullpath = string(oldEntry.FullPath)
} else if newEntry != nil {
fullpath = string(newEntry.FullPath)
} else {
return nil
}
// println("fullpath:", fullpath)
if strings.HasPrefix(fullpath, SystemLogDir) {
// No event covers this write: release its stamp.
if newEntry != nil {
f.metaLogInflight.release(newEntry.CommitTsNs)
}
return nil
}
foundSelf := false
for _, sig := range signatures {
if sig == f.Signature {
foundSelf = true
}
}
if !foundSelf {
signatures = append(signatures, f.Signature)
}
event := f.newMetadataEvent(oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures, f.stampOf(newEntry))
// Clear the stamp after the buffer append below - deliberately also on
// append failure (see the metaLogInflight comment).
defer f.metaLogInflight.done(event.TsNs)
eventNotification := event.EventNotification
if notification.Queue != nil {
glog.V(3).Infof("notifying entry update %v", fullpath)
if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil {
// throw message
glog.Error(err)
}
}
f.logMetaEvent(ctx, event)
if sink := metadataEventSinkFromContext(ctx); sink != nil {
sink.Record(event)
}
f.onMetadataChangeEvent(event)
return event
}
// metaLogInflight tracks events stamped but not yet appended to the local
// log buffer - the two are separated by notification work that can block,
// and a claim ignoring that window would assert durability or delivery for
// timestamps still on their way in. Stamping shares the reader's lock, so an
// event is always visible here or (bumped monotonically) in the buffer.
//
// An event whose append fails also clears its stamp: it is dropped from the
// change stream entirely (loudly logged there), and a watermark waiting for
// it would pin this filer's claims forever.
type metaLogInflight struct {
sync.Mutex
stamped map[int64]int
lastStampNs int64
lastClaimNs int64
}
// stamp assigns the event timestamp and registers it as in flight. Stamps
// are monotonic against the registry's own history and every issued claim,
// so a wall-clock step backwards cannot slip a new stamp under a floor or
// watermark already handed out.
func (t *metaLogInflight) stamp() int64 {
t.Lock()
defer t.Unlock()
floor := t.lastStampNs
if t.lastClaimNs > floor {
floor = t.lastClaimNs
}
tsNs := time.Now().UnixNano()
if tsNs <= floor {
tsNs = floor + 1
}
t.lastStampNs = tsNs
if t.stamped == nil {
t.stamped = make(map[int64]int)
}
t.stamped[tsNs]++
return tsNs
}
// done removes a stamp once the event has been appended to the buffer.
func (t *metaLogInflight) done(tsNs int64) {
t.Lock()
defer t.Unlock()
if t.stamped[tsNs] <= 1 {
delete(t.stamped, tsNs)
} else {
t.stamped[tsNs]--
}
}
// inFlight reports whether tsNs is a pre-commit stamp still awaiting its
// buffer append — how an event adopts the stamp an entry carries.
func (t *metaLogInflight) inFlight(tsNs int64) bool {
t.Lock()
defer t.Unlock()
return t.stamped[tsNs] > 0
}
// release drops a stamp whose event will never be appended.
func (t *metaLogInflight) release(tsNs int64) {
if tsNs == 0 {
return
}
t.Lock()
defer t.Unlock()
if t.stamped[tsNs] <= 1 {
delete(t.stamped, tsNs)
} else {
t.stamped[tsNs]--
}
}
// minTsNs returns the oldest in-flight stamp, or 0 when nothing is in flight.
func (t *metaLogInflight) minTsNs() int64 {
t.Lock()
defer t.Unlock()
var min int64
for tsNs := range t.stamped {
if min == 0 || tsNs < min {
min = tsNs
}
}
return min
}
// claimThrough caps a completeness claim by the oldest in-flight stamp and
// fences it: later stamps always land above the returned claim, so a wall
// clock stepping backwards cannot slide a new event under a watermark a
// peer has already advanced to.
func (t *metaLogInflight) claimThrough(nowNs int64) int64 {
t.Lock()
defer t.Unlock()
claim := nowNs
for tsNs := range t.stamped {
if tsNs-1 < claim {
claim = tsNs - 1
}
}
if claim > t.lastClaimNs {
t.lastClaimNs = claim
}
return claim
}
// LocalFlushedThroughTsNs reports the timestamp through which the local meta
// log is durably on disk: everything at or below it is appended and flushed,
// and nothing can land at or below it later. The registry is consulted before
// the buffer: an event already appended is visible to the buffer claim, one
// still in flight caps the claim, and one stamped later is fenced above it.
func (f *Filer) LocalFlushedThroughTsNs(nowNs int64) int64 {
claim := f.metaLogInflight.claimThrough(nowNs)
if buffered := f.LocalMetaLogBuffer.FlushedThroughTsNs(nowNs); buffered < claim {
claim = buffered
}
return claim
}
// LocalDeliveredThroughTsNs caps a delivery-freshness claim (an idle
// heartbeat's timestamp, a listing snapshot) by the oldest in-flight stamp: a
// stamped-but-unappended event has not been streamed to anyone, nor is its
// write certain to be visible yet, and a peer aggregator turns the claim into
// its delivery low-watermark.
func (f *Filer) LocalDeliveredThroughTsNs(nowNs int64) int64 {
return f.metaLogInflight.claimThrough(nowNs)
}
// StampMetaLogInflightForTest and DoneMetaLogInflightForTest let tests in
// other packages exercise the claim caps. Not for production use.
func (f *Filer) StampMetaLogInflightForTest() int64 {
return f.metaLogInflight.stamp()
}
func (f *Filer) DoneMetaLogInflightForTest(tsNs int64) {
f.metaLogInflight.done(tsNs)
}
// stampEntryCommitTsNs assigns the mutation's event timestamp to the entry
// pre-write so the emitted event and stored version carry the same commit
// time. The stamp stays in flight until the event appends; a write no event
// covers must release it via releaseEntryCommitTsNs.
func (f *Filer) stampEntryCommitTsNs(entry *Entry) int64 {
tsNs := f.metaLogInflight.stamp()
entry.CommitTsNs = tsNs
return tsNs
}
// releaseEntryCommitTsNs drops a stamp no event will adopt.
func (f *Filer) releaseEntryCommitTsNs(tsNs int64) {
f.metaLogInflight.release(tsNs)
}
// StampEntryCommitTsNs/ReleaseEntryCommitTsNs expose the lifecycle to the
// filer server's mutation paths.
func (f *Filer) StampEntryCommitTsNs(entry *Entry) int64 {
return f.stampEntryCommitTsNs(entry)
}
func (f *Filer) ReleaseEntryCommitTsNs(tsNs int64) {
f.metaLogInflight.release(tsNs)
}
// stampOf returns the event's timestamp: the entry's in-flight stamp, or a
// fresh one for events with no newly stamped entry (deletes).
func (f *Filer) stampOf(newEntry *Entry) int64 {
if newEntry != nil && f.metaLogInflight.inFlight(newEntry.CommitTsNs) {
return newEntry.CommitTsNs
}
return f.metaLogInflight.stamp()
}
func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32, tsNs int64) *filer_pb.SubscribeMetadataResponse {
if oldEntry == nil && newEntry == nil {
return nil
}
var fullpath util.FullPath
if oldEntry != nil {
fullpath = oldEntry.FullPath
}
if fullpath == "" && newEntry != nil {
fullpath = newEntry.FullPath
}
dir, _ := fullpath.DirAndName()
newParentPath := ""
if newEntry != nil {
newParentPath, _ = newEntry.FullPath.DirAndName()
}
return &filer_pb.SubscribeMetadataResponse{
Directory: dir,
EventNotification: &filer_pb.EventNotification{
OldEntry: oldEntry.ToProtoEntry(),
NewEntry: newEntry.ToProtoEntry(),
DeleteChunks: deleteChunks,
NewParentPath: newParentPath,
IsFromOtherCluster: isFromOtherCluster,
Signatures: signatures,
},
// In flight until appended to the local log buffer (see metaLogInflight).
TsNs: tsNs,
MutationTsNs: tsNs,
}
}
// notifyRenameTarget wakes the subscribers of a move's destination directory,
// which the partition key (the source directory) does not cover.
func notifyRenameTarget(logBuffer *log_buffer.LogBuffer, event *filer_pb.SubscribeMetadataResponse) {
if target := filer_pb.MetadataEventTargetDirectory(event); target != event.Directory {
logBuffer.NotifySubscribers([]byte(target))
}
}
func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) {
data, err := proto.Marshal(event)
if err != nil {
glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %s -> %s ts_ns=%d: %v", filer_pb.MetadataEventSourceFullPath(event), filer_pb.MetadataEventTargetFullPath(event), event.TsNs, err)
return
}
tsNs, err := f.LocalMetaLogBuffer.AddDataToBufferAndGetTs([]byte(event.Directory), data, event.TsNs)
if err != nil {
glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err)
return
}
notifyRenameTarget(f.LocalMetaLogBuffer, event)
// The in-memory copy now carries the position the stream assigns it, the
// same value subscribers will see delivered.
event.TsNs = tsNs
}
// metadataLogUploadLimit is the piece size a metadata log flush starts with. A
// volume server refuses anything over its -fileSizeLimitMB (256 MB by default),
// and a single oversized event — a CreateEntry carrying a large inline Content,
// say — grows the log buffer well past that, leaving a blob that can never be
// written and blocks every later flush behind it. BufferSize is what an
// ordinary flush already produces, so it is a size the volume server accepts
// under any configuration that works at all; a cluster running below it says so
// in the rejection and volumeFileSizeLimit picks the real limit up from there.
const metadataLogUploadLimit = log_buffer.BufferSize
// shutdownMetadataLogFlushBudget bounds how long metadata-log flushes may keep
// retrying once the filer is shutting down; Shutdown cancels the shared flush
// context when it expires.
const shutdownMetadataLogFlushBudget = 15 * time.Second
var fileSizeLimitPattern = regexp.MustCompile(`file over the limited (\d+) bytes`)
// volumeFileSizeLimit reads the byte limit back out of a volume server's size
// rejection, and returns 0 for any other error.
func volumeFileSizeLimit(err error) int {
match := fileSizeLimitPattern.FindStringSubmatch(err.Error())
if match == nil {
return 0
}
limit, convErr := strconv.Atoi(match[1])
if convErr != nil {
return 0
}
return limit
}
func (f *Filer) logFlushFunc(logBuffer *log_buffer.LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
if len(buf) == 0 {
return
}
startTime, stopTime = startTime.UTC(), stopTime.UTC()
targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir,
startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFilerId,
// startTime.Second(), startTime.Nanosecond(),
)
// One piece at a time, each retried on its own so a partial success is not
// replayed, and the piece size follows the limit the volume servers report.
// While the filer keeps running the retry is unbounded so no metadata is
// dropped; once Shutdown arms the flush deadline the shared context cancels
// and the flush drops what is left instead of holding shutdown open.
limit := metadataLogUploadLimit
ctx := f.flushCtx
if ctx == nil {
ctx = context.Background()
}
for len(buf) > 0 {
piece := nextLogPiece(buf, limit)
if err := f.appendToFile(ctx, targetFile, piece); err != nil {
glog.V(0).Infof("metadata log write failed %s: %v", targetFile, err)
if reported := volumeFileSizeLimit(err); reported > 0 && reported < limit {
glog.V(0).Infof("metadata log upload limit lowered to %d bytes", reported)
limit = reported
continue
}
select {
case <-ctx.Done():
logBuffer.NoteFlushDropped(len(buf))
glog.V(0).Infof("metadata log flush abandoned during shutdown, %d bytes left for %s: %v", len(buf), targetFile, ctx.Err())
return
case <-time.After(737 * time.Millisecond):
}
continue
}
buf = buf[len(piece):]
}
}
// nextLogPiece returns the leading piece of a flushed log buffer, at most
// maxSize bytes and ending on a record boundary where it can so the piece still
// decodes on its own. A record longer than maxSize is cut by size instead; the
// readers fall back to streaming the whole file when a chunk does not decode
// standalone, so a record may cross a chunk boundary.
func nextLogPiece(buf []byte, maxSize int) []byte {
if len(buf) <= maxSize {
return buf
}
pos := 0
for pos+4 <= len(buf) {
size := int(util.BytesToUint32(buf[pos : pos+4]))
end := pos + 4 + size
if size <= 0 || end > len(buf) || end > maxSize {
break
}
pos = end
}
if pos == 0 {
// Either the leading record alone is over the limit, or buf starts
// mid-record because the piece before it was cut by size.
return buf[:maxSize]
}
return buf[:pos]
}
var (
volumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`)
chunkNotFoundPattern = regexp.MustCompile(`(urls not found|File Not Found)`)
httpNotFoundPattern = regexp.MustCompile(`404 Not Found: not found`)
)
// isChunkNotFoundError checks if the error indicates that a volume or chunk
// has been deleted and is no longer available. These errors can be skipped
// when reading persisted log files since the data is unrecoverable.
func isChunkNotFoundError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, util_http.ErrNotFound) || errors.Is(err, nethttp.ErrMissingFile) {
return true
}
errMsg := err.Error()
return volumeNotFoundPattern.MatchString(errMsg) ||
chunkNotFoundPattern.MatchString(errMsg) ||
httpNotFoundPattern.MatchString(errMsg)
}
// persistedLogReplayLimit caps concurrent legacy replays; decodes are shared
// through the persisted-log cache, so this only bounds the listing fan-out.
const persistedLogReplayLimit = 64
var persistedLogReplaySem = make(chan struct{}, persistedLogReplayLimit)
func (f *Filer) ReadPersistedLogBuffer(ctx context.Context, startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, isDone bool, err error) {
// Cap concurrent replays; bail if the stream is already gone so cancelled
// clients do not park on the semaphore.
if err := ctx.Err(); err != nil {
return 0, false, err
}
select {
case persistedLogReplaySem <- struct{}{}:
defer func() { <-persistedLogReplaySem }()
case <-ctx.Done():
return 0, false, ctx.Err()
}
visitor, visitErr := f.collectPersistedLogBuffer(startPosition, stopTsNs)
if visitErr != nil {
if visitErr == io.EOF {
return
}
err = fmt.Errorf("reading from persisted logs: %w", visitErr)
return
}
// Readahead: run the visitor in a background goroutine so volume server I/O
// for the next log file overlaps with event processing and gRPC delivery.
const readaheadSize = 8192
type entryOrErr struct {
entry *filer_pb.LogEntry
err error
}
ch := make(chan entryOrErr, readaheadSize)
stopReadahead := make(chan struct{})
readaheadDone := make(chan struct{})
go func() {
defer close(ch)
defer close(readaheadDone)
for {
entry, readErr := visitor.GetNext()
if readErr != nil {
if readErr != io.EOF {
select {
case ch <- entryOrErr{err: fmt.Errorf("read next from persisted logs: %w", readErr)}:
case <-stopReadahead:
}
}
return
}
select {
case ch <- entryOrErr{entry: entry}:
case <-stopReadahead:
return
}
}
}()
// Stop the readahead goroutine, wait for it to exit, then release any log
// file readers it left open (e.g. on early return or cancellation).
defer func() {
close(stopReadahead)
<-readaheadDone
visitor.Close()
}()
for item := range ch {
if item.err != nil {
err = item.err
return
}
var processErr error
isDone, processErr = eachLogEntryFn(item.entry)
if processErr != nil {
err = fmt.Errorf("process persisted log entry: %w", processErr)
return
}
lastTsNs = item.entry.TsNs
if isDone {
return
}
}
return
}