seaweedfs/weed/command/filer_sync_jobs.go

package command

import (
	"container/heap"
	"path"
	"sync"
	"sync/atomic"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// maxFailedSyncEvents bounds the failedTs ledger. A destination rejecting
// every event would otherwise add an entry per source event for the life of
// the processor.
var maxFailedSyncEvents = 1 << 16

// tsMinHeap implements heap.Interface for int64 timestamps.
type tsMinHeap []int64

func (h tsMinHeap) Len() int           { return len(h) }
func (h tsMinHeap) Less(i, j int) bool { return h[i] < h[j] }
func (h tsMinHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h *tsMinHeap) Push(x any)        { *h = append(*h, x.(int64)) }
func (h *tsMinHeap) Pop() any {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[:n-1]
	return x
}

// jobKind classifies a sync job for conflict detection. Directory events are
// split into "barrier" (create/delete/rename) and "non-barrier" (in-place
// attribute update) so that attribute-only directory updates — which do not
// reshape the namespace — no longer serialize every file operation in the
// subtree.
type jobKind int

const (
	// kindFile is a regular file event.
	kindFile jobKind = iota
	// kindBarrierDir is a directory create, delete, or rename. It acts as a
	// subtree barrier: it waits for all active descendants to drain, and it
	// blocks every event under it from being admitted until it completes.
	kindBarrierDir
	// kindNonBarrierDir is a directory attribute update (mtime/xattr/chmod
	// with the same parent and name). It does not block descendants and is
	// not blocked by ancestor directories, but it still bumps the ancestor
	// descendant counters so an incoming barrier dir on an ancestor path
	// still waits for it to drain.
	kindNonBarrierDir
)

type syncJobPaths struct {
	path     util.FullPath
	newPath  util.FullPath // empty for non-renames
	kind     jobKind
	dataSize int64
}

// failedEventKey identifies an event for the failure ledger. A timestamp alone
// is not unique across events, so a success for one event must not clear an
// unresolved failure recorded for a different event at the same TsNs.
type failedEventKey struct {
	tsNs    int64
	path    util.FullPath
	newPath util.FullPath
	kind    jobKind
}

// syncStreamMetrics holds the metric children for one sync stream, curried
// once so per-event updates skip the label lookup.
type syncStreamMetrics struct {
	received       prometheus.Counter
	processed      prometheus.Counter
	failed         prometheus.Counter
	inFlight       prometheus.Gauge
	receivedBytes  prometheus.Counter
	processedBytes prometheus.Counter
	failedBytes    prometheus.Counter
	inFlightBytes  prometheus.Gauge
}

type MetadataProcessor struct {
	// activeJobCount is the number of in-flight jobs and activeJobTs counts
	// them per event timestamp. Several events can share a TsNs — batched
	// writes log together — so a single slot per TsNs would drain early and
	// let the resubscribe or the watermark outrun a sibling still running.
	activeJobCount       int
	activeJobTs          map[int64]int
	activeJobsLock       sync.Mutex
	activeJobsCond       *sync.Cond
	concurrencyLimit     int
	fn                   pb.ProcessMetadataFunc
	processedTsWatermark atomic.Int64
	filteredTsNs         int64

	// Indexes for O(depth) conflict detection, replacing O(n) linear scan.
	// activeFilePaths counts active file jobs at each exact path.
	activeFilePaths map[util.FullPath]int
	// activeBarrierDirPaths counts active barrier-dir jobs at each exact
	// path. Only barrier dirs are tracked here; non-barrier dir updates are
	// deliberately invisible to the ancestor check so that they don't
	// serialize every file descendant.
	activeBarrierDirPaths map[util.FullPath]int
	// activeNonBarrierDirPaths counts active non-barrier dir jobs at each
	// exact path. This is read *only* by incoming barrier dirs, so a
	// delete/rename/create at p correctly waits for an in-flight chmod/
	// xattr/mtime update at the same p. It is deliberately invisible to the
	// ancestor check, so non-barrier updates still don't serialize file
	// descendants.
	activeNonBarrierDirPaths map[util.FullPath]int
	// descendantCount counts active jobs (of any kind) strictly under each
	// directory. Read by incoming barrier dirs so they wait for their whole
	// subtree to drain before running, regardless of descendant kind.
	descendantCount map[util.FullPath]int

	// tsHeap is a min-heap of active job timestamps with lazy deletion,
	// used for O(log n) amortized watermark tracking.
	tsHeap tsMinHeap

	// failedTs records every event whose job returned an error and has not
	// since completed, and oldestFailedTsNs caches its minimum (0 when empty).
	// The watermark is never advanced to it or past it, so the persisted sync
	// offset stays behind the failure and a restart replays the event instead
	// of skipping it forever. Past maxFailedSyncEvents the set collapses to a
	// sticky pin at the smallest failure seen: replay from the oldest failure
	// still works, but individual recoveries no longer unpin until a restart.
	failedTs         map[failedEventKey]struct{}
	failedSticky     bool
	oldestFailedTsNs int64

	// resubscribeCh closes once a failure has stopped the processor and all
	// in-flight jobs have drained, asking the metadata follower to drop the
	// stream so the caller's reconnect replays the pinned events in order —
	// and never races the replay against work still running in this abandoned
	// processor.
	resubscribeCh   chan struct{}
	resubscribeOnce sync.Once

	// stopped latches when a job failure pins the watermark. Admission then
	// drops new events instead of queueing them into a processor that is about
	// to be abandoned: every skipped event replays from the pinned watermark
	// after the reconnect, and on a stream that never goes quiet the drain —
	// and with it the replay — would otherwise never come. The cond broadcast
	// releases a blocked AddSyncJob. A redelivery of an event still in
	// failedTs is the one exception: it may run so its success shrinks the
	// replay.
	stopped bool

	// metrics is nil for callers that do not report per-event metrics.
	metrics *syncStreamMetrics
}

func NewMetadataProcessor(fn pb.ProcessMetadataFunc, concurrency int, offsetTsNs int64) *MetadataProcessor {
	t := &MetadataProcessor{
		fn:                       fn,
		activeJobTs:              make(map[int64]int),
		concurrencyLimit:         concurrency,
		activeFilePaths:          make(map[util.FullPath]int),
		activeBarrierDirPaths:    make(map[util.FullPath]int),
		activeNonBarrierDirPaths: make(map[util.FullPath]int),
		descendantCount:          make(map[util.FullPath]int),
		failedTs:                 make(map[failedEventKey]struct{}),
		resubscribeCh:            make(chan struct{}),
	}
	t.processedTsWatermark.Store(offsetTsNs)
	t.activeJobsCond = sync.NewCond(&t.activeJobsLock)
	return t
}

// SetMetrics enables per-event metrics for this stream, labeled the same way
// as the existing sync_offset gauge.
func (t *MetadataProcessor) SetMetrics(sourceFiler, targetFiler, clientName, path string) {
	t.metrics = &syncStreamMetrics{
		received:       statsCollect.FilerSyncEventsReceivedCounter.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		processed:      statsCollect.FilerSyncEventsProcessedCounter.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		failed:         statsCollect.FilerSyncEventsFailedCounter.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		inFlight:       statsCollect.FilerSyncInFlightJobsGauge.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		receivedBytes:  statsCollect.FilerSyncReceivedBytesCounter.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		processedBytes: statsCollect.FilerSyncProcessedBytesCounter.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		failedBytes:    statsCollect.FilerSyncFailedBytesCounter.WithLabelValues(sourceFiler, targetFiler, clientName, path),
		inFlightBytes:  statsCollect.FilerSyncInFlightBytesGauge.WithLabelValues(sourceFiler, targetFiler, clientName, path),
	}
}

// OldestFailedTsNs returns the timestamp of the oldest permanently failed
// event, or 0 when none is pinning the watermark.
func (t *MetadataProcessor) OldestFailedTsNs() int64 {
	t.activeJobsLock.Lock()
	defer t.activeJobsLock.Unlock()
	return t.oldestFailedTsNs
}

// ResubscribeCh closes once a job failure has stopped the processor and its
// in-flight jobs have drained, signaling the metadata follower to drop the
// stream so a reconnect replays what the watermark still covers.
func (t *MetadataProcessor) ResubscribeCh() <-chan struct{} {
	return t.resubscribeCh
}

// pathAncestors returns all proper ancestor directories of p.
// For "/a/b/c", returns ["/a/b", "/a", "/"].
func pathAncestors(p util.FullPath) []util.FullPath {
	var ancestors []util.FullPath
	s := string(p)
	for {
		parent := path.Dir(s)
		if parent == s {
			break
		}
		ancestors = append(ancestors, util.FullPath(parent))
		s = parent
	}
	return ancestors
}

// addPathToIndex registers a path in the conflict detection indexes.
// Must be called under activeJobsLock.
func (t *MetadataProcessor) addPathToIndex(p util.FullPath, kind jobKind) {
	switch kind {
	case kindFile:
		t.activeFilePaths[p]++
	case kindBarrierDir:
		t.activeBarrierDirPaths[p]++
	case kindNonBarrierDir:
		t.activeNonBarrierDirPaths[p]++
	}
	for _, ancestor := range pathAncestors(p) {
		t.descendantCount[ancestor]++
	}
}

// removePathFromIndex unregisters a path from the conflict detection indexes.
// Must be called under activeJobsLock.
func (t *MetadataProcessor) removePathFromIndex(p util.FullPath, kind jobKind) {
	switch kind {
	case kindFile:
		if t.activeFilePaths[p] <= 1 {
			delete(t.activeFilePaths, p)
		} else {
			t.activeFilePaths[p]--
		}
	case kindBarrierDir:
		if t.activeBarrierDirPaths[p] <= 1 {
			delete(t.activeBarrierDirPaths, p)
		} else {
			t.activeBarrierDirPaths[p]--
		}
	case kindNonBarrierDir:
		if t.activeNonBarrierDirPaths[p] <= 1 {
			delete(t.activeNonBarrierDirPaths, p)
		} else {
			t.activeNonBarrierDirPaths[p]--
		}
	}
	for _, ancestor := range pathAncestors(p) {
		if t.descendantCount[ancestor] <= 1 {
			delete(t.descendantCount, ancestor)
		} else {
			t.descendantCount[ancestor]--
		}
	}
}

// pathConflicts checks if a single path conflicts with any active job.
// Conflict rules:
//   - any kind vs same-path barrier dir: wait (a create/delete/rename on p
//     must fully serialize against any other operation touching p, including
//     non-barrier attribute updates and files at the same path)
//   - incoming barrier dir vs same-path non-barrier dir update: wait (a
//     delete/rename/create on p must wait for an in-flight chmod/xattr/mtime
//     update at the same p to drain)
//   - file vs same-path file: wait
//   - file vs same-path barrier dir: wait (covered by the barrier-at-p check
//     above; also serializes a file-to-dir / dir-to-file promotion)
//   - barrier dir vs same-path file: wait
//   - barrier dir vs any descendant (file or dir, barrier or not): wait
//   - barrier ancestor: always wait, regardless of incoming kind
//   - non-barrier dir vs descendants: never conflicts
//   - non-barrier dir vs same-path non-barrier dir: never conflicts (attribute
//     bumps are "last writer wins"; this intentionally lets rapid mtime /
//     xattr updates overlap)
func (t *MetadataProcessor) pathConflicts(p util.FullPath, kind jobKind) bool {
	// A barrier dir in flight at p serializes every new job at p. This is the
	// strictest same-path rule and applies regardless of incoming kind.
	if t.activeBarrierDirPaths[p] > 0 {
		return true
	}
	// An incoming barrier dir must also wait for any in-flight non-barrier
	// dir update at the same path. Without this check, a delete or rename on
	// a directory could overlap with an attribute bump in progress for the
	// same directory.
	if kind == kindBarrierDir && t.activeNonBarrierDirPaths[p] > 0 {
		return true
	}
	// A file in flight at p blocks new file or barrier-dir jobs at p. A
	// non-barrier dir update at p is allowed through — by construction files
	// and dirs at the same path only coexist across a promotion, which is a
	// barrier event handled by the check above.
	if t.activeFilePaths[p] > 0 && (kind == kindFile || kind == kindBarrierDir) {
		return true
	}
	// Barrier dirs additionally wait for their whole in-flight subtree.
	if kind == kindBarrierDir && t.descendantCount[p] > 0 {
		return true
	}
	// Any barrier dir on a proper ancestor blocks everything under it.
	for _, ancestor := range pathAncestors(p) {
		if t.activeBarrierDirPaths[ancestor] > 0 {
			return true
		}
	}
	return false
}

func (t *MetadataProcessor) conflictsWith(resp *filer_pb.SubscribeMetadataResponse) bool {
	p, newPath, kind := extractJobInfo(resp)
	if t.pathConflicts(p, kind) {
		return true
	}
	if newPath != "" && t.pathConflicts(newPath, kind) {
		return true
	}
	return false
}

func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) {
	if filer_pb.IsEmpty(resp) {
		// A filtered-progress marker means the source skipped everything below
		// it for us; once all earlier work has finished, the watermark can move
		// to it so idle stretches still advance the resume point.
		t.activeJobsLock.Lock()
		defer t.activeJobsLock.Unlock()
		if resp.TsNs > t.filteredTsNs {
			t.filteredTsNs = resp.TsNs
		}
		if t.activeJobCount == 0 && resp.TsNs > t.processedTsWatermark.Load() &&
			(t.oldestFailedTsNs == 0 || resp.TsNs < t.oldestFailedTsNs) {
			t.processedTsWatermark.Store(resp.TsNs)
		}
		return
	}

	dataSize := eventDataSize(resp)

	t.activeJobsLock.Lock()
	defer t.activeJobsLock.Unlock()

	p, newPath, kind := extractJobInfo(resp)
	eventKey := failedEventKey{tsNs: resp.TsNs, path: p, newPath: newPath, kind: kind}

	for t.activeJobCount >= t.concurrencyLimit || t.conflictsWith(resp) {
		// A stopped processor never queues: an event that cannot start drops
		// and replays in order after the resubscribe.
		if t.stopped {
			return
		}
		t.activeJobsCond.Wait()
	}
	if t.stopped {
		select {
		case <-t.resubscribeCh:
			// Already drained and signaled: nothing may start now, or it would
			// race the replay the signal just asked for.
			return
		default:
		}
		// The one event still worth running is a failure still pinning the
		// watermark, redelivered on this stream: its success clears the ledger
		// entry and shrinks the replay. Everything else replays anyway.
		if _, pinned := t.failedTs[eventKey]; !pinned {
			return
		}
	}

	// counted once admitted: received-processed-failed is the number of events
	// this processor read but has not finished. A dropped event is not counted
	// here — the replay's own admission counts it.
	if t.metrics != nil {
		t.metrics.received.Inc()
		t.metrics.receivedBytes.Add(float64(dataSize))
	}

	jobPaths := &syncJobPaths{path: p, newPath: newPath, kind: kind, dataSize: dataSize}

	t.activeJobCount++
	t.activeJobTs[resp.TsNs]++
	t.addPathToIndex(p, kind)
	if newPath != "" {
		t.addPathToIndex(newPath, kind)
	}
	// Inc/Dec rather than Set from local state: after a subscription retry a
	// new processor shares these children with the old one's still-draining
	// jobs, and each job accounting for itself keeps the total truthful.
	if t.metrics != nil {
		t.metrics.inFlight.Inc()
		t.metrics.inFlightBytes.Add(float64(dataSize))
	}

	heap.Push(&t.tsHeap, resp.TsNs)

	go func() {

		jobErr := util.Retry("metadata processor", func() error {
			return t.fn(resp)
		})

		t.activeJobsLock.Lock()
		defer t.activeJobsLock.Unlock()

		failedKey := failedEventKey{tsNs: resp.TsNs, path: jobPaths.path, newPath: jobPaths.newPath, kind: jobPaths.kind}
		if jobErr != nil {
			// Latch the stop the moment a failure lands: events behind the pin
			// replay after the resubscribe anyway, and a stream that never goes
			// quiet would otherwise keep the active count above zero so the
			// failure stays pinned until an unrelated reconnect — the wait this
			// whole mechanism exists to remove.
			t.stopped = true
			t.activeJobsCond.Broadcast()
			if t.failedSticky {
				if resp.TsNs < t.oldestFailedTsNs {
					t.oldestFailedTsNs = resp.TsNs
				}
				glog.Errorf("process %v: %v", resp, jobErr)
			} else if _, recorded := t.failedTs[failedKey]; !recorded {
				if len(t.failedTs) >= maxFailedSyncEvents {
					t.failedSticky = true
					t.failedTs = nil
					if resp.TsNs < t.oldestFailedTsNs {
						t.oldestFailedTsNs = resp.TsNs
					}
					glog.Warningf("process %v: %v; over %d unresolved failures, pinning sync offset at %v until restart", resp, jobErr, maxFailedSyncEvents, time.Unix(0, t.oldestFailedTsNs))
				} else {
					t.failedTs[failedKey] = struct{}{}
					if t.oldestFailedTsNs == 0 || resp.TsNs < t.oldestFailedTsNs {
						t.oldestFailedTsNs = resp.TsNs
						glog.Errorf("process %v: %v; holding sync offset at %v so this event is replayed on restart", resp, jobErr, time.Unix(0, resp.TsNs))
					} else {
						glog.Errorf("process %v: %v", resp, jobErr)
					}
				}
			}
		} else if _, recorded := t.failedTs[failedKey]; recorded {
			delete(t.failedTs, failedKey)
			if resp.TsNs == t.oldestFailedTsNs {
				t.oldestFailedTsNs = 0
				for k := range t.failedTs {
					if t.oldestFailedTsNs == 0 || k.tsNs < t.oldestFailedTsNs {
						t.oldestFailedTsNs = k.tsNs
					}
				}
			}
		}

		t.activeJobCount--
		t.activeJobTs[resp.TsNs]--
		if t.activeJobTs[resp.TsNs] == 0 {
			delete(t.activeJobTs, resp.TsNs)
		}
		t.removePathFromIndex(jobPaths.path, jobPaths.kind)
		if jobPaths.newPath != "" {
			t.removePathFromIndex(jobPaths.newPath, jobPaths.kind)
		}
		if t.metrics != nil {
			if jobErr != nil {
				t.metrics.failed.Inc()
				t.metrics.failedBytes.Add(float64(jobPaths.dataSize))
			} else {
				t.metrics.processed.Inc()
				t.metrics.processedBytes.Add(float64(jobPaths.dataSize))
			}
			t.metrics.inFlight.Dec()
			t.metrics.inFlightBytes.Sub(float64(jobPaths.dataSize))
		}

		// Lazy-clean stale entries from heap top (already-completed jobs).
		// Each entry is pushed once and popped once: O(log n) amortized.
		for t.tsHeap.Len() > 0 {
			if t.activeJobTs[t.tsHeap[0]] > 0 {
				break
			}
			heap.Pop(&t.tsHeap)
		}
		// If this was the oldest job, advance the watermark, but never to or
		// past an event that failed: the offset is the durable resume point,
		// and moving it over a failure drops that event for good.
		if t.tsHeap.Len() == 0 || resp.TsNs < t.tsHeap[0] {
			if t.oldestFailedTsNs == 0 || resp.TsNs < t.oldestFailedTsNs {
				t.processedTsWatermark.Store(resp.TsNs)
			}
		}
		if t.activeJobCount == 0 && t.filteredTsNs > t.processedTsWatermark.Load() &&
			(t.oldestFailedTsNs == 0 || t.filteredTsNs < t.oldestFailedTsNs) {
			t.processedTsWatermark.Store(t.filteredTsNs)
		}
		// Signal once the stop has drained: even if a redelivery cleared the
		// pin, events dropped while stopped still have to replay.
		if t.stopped && t.activeJobCount == 0 {
			t.resubscribeOnce.Do(func() { close(t.resubscribeCh) })
		}
		t.activeJobsCond.Signal()
	}()
}

// eventDataSize is the chunk data this event will copy: chunks on the new
// entry that the old entry does not already have. Deletes, renames, and
// attribute-only updates all come out zero, so byte rates reflect data
// movement rather than metadata churn.
func eventDataSize(resp *filer_pb.SubscribeMetadataResponse) (size int64) {
	message := resp.EventNotification
	if message.NewEntry == nil {
		return 0
	}
	newChunks := message.NewEntry.GetChunks()
	if message.OldEntry != nil {
		newChunks = filer.DoMinusChunks(newChunks, message.OldEntry.GetChunks())
	}
	for _, chunk := range newChunks {
		size += int64(chunk.Size)
	}
	return size
}

// extractJobInfo derives the conflict-detection path(s) and job kind for a
// metadata event. A rename returns both the source and destination paths; all
// other event shapes return only the primary path.
func extractJobInfo(resp *filer_pb.SubscribeMetadataResponse) (p, newPath util.FullPath, kind jobKind) {
	oldEntry := resp.EventNotification.OldEntry
	newEntry := resp.EventNotification.NewEntry
	// create
	if filer_pb.IsCreate(resp) {
		p = util.FullPath(resp.Directory).Child(newEntry.Name)
		kind = classifyDirEvent(newEntry.IsDirectory, false)
		return
	}
	if filer_pb.IsDelete(resp) {
		p = util.FullPath(resp.Directory).Child(oldEntry.Name)
		kind = classifyDirEvent(oldEntry.IsDirectory, false)
		return
	}
	if filer_pb.IsUpdate(resp) {
		p = util.FullPath(resp.Directory).Child(newEntry.Name)
		// In-place attribute update: non-barrier when the entry is a dir.
		kind = classifyDirEvent(newEntry.IsDirectory, true)
		return
	}
	// renaming: the namespace is reshaped on both sides, so a directory
	// rename is a barrier on both source and destination.
	p = util.FullPath(resp.Directory).Child(oldEntry.Name)
	newPath = util.FullPath(resp.EventNotification.NewParentPath).Child(newEntry.Name)
	kind = classifyDirEvent(oldEntry.IsDirectory, false)
	return
}

// classifyDirEvent maps an entry's (isDirectory, isAttributeUpdate) pair to a
// jobKind. Attribute-only updates on directories are the only non-barrier
// case; everything else on a directory (create/delete/rename) is a barrier,
// and everything on a file is kindFile.
func classifyDirEvent(isDirectory, isAttributeUpdate bool) jobKind {
	if !isDirectory {
		return kindFile
	}
	if isAttributeUpdate {
		return kindNonBarrierDir
	}
	return kindBarrierDir
}