seaweedfs/weed/filer/filer_remote_tombstone.go

package filer

import (
	"context"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
	"google.golang.org/protobuf/proto"
)

const (
	// defaultRemoteDeletionTombstoneTTL bounds a tombstone when no write-back
	// sync offset ever confirms the remote delete. A remote object re-created
	// outside the filer at a deleted path stays hidden for this long.
	defaultRemoteDeletionTombstoneTTL = 24 * time.Hour
	// defaultRemoteDeletionTombstoneLimit bounds tracked paths; at the cap a
	// new delete is not recorded and the lazy fetch stays off instead.
	defaultRemoteDeletionTombstoneLimit = 1 << 16
)

// Variables for tests that fill or age the table.
var (
	remoteDeletionTombstoneTTL   = defaultRemoteDeletionTombstoneTTL
	remoteDeletionTombstoneLimit = defaultRemoteDeletionTombstoneLimit
)

// remoteDeletionTombstones tracks paths deleted under a remote mount whose
// remote object may still exist because the write-back daemon has not
// consumed the delete event yet. A lazy remote fetch or listing must not
// resurrect them. A tombstone lifts when the path is written again, when the
// mount's persisted sync offset passes the delete event, or on TTL.
//
// Tombstones recorded before their metadata event lands (the synchronous
// delete path) are marked pending: the sync offset orders against event
// timestamps, which only the event itself knows, so a pending tombstone can
// only be lifted by the event confirming it or by TTL. Once the event stamps
// the real timestamp the tombstone is releasable by the offset.
// remoteDeletionTombstone carries the delete's commit stamp — the basis for
// ordering it against other mutations — and, once the event lands, the local
// stream position it was delivered under, which the write-back offset must
// pass before the remote delete counts as consumed.
type remoteDeletionTombstone struct {
	commitTsNs int64
	streamTsNs int64
}

type remoteDeletionTombstones struct {
	mu      sync.Mutex
	files   map[string]remoteDeletionTombstone // file path -> delete event
	dirs    map[string]remoteDeletionTombstone // deleted directory path -> event; covers its subtree
	pending map[string]bool                    // tombstone path recorded ahead of its event
	// full is set when a delete could not be recorded: from then on an
	// unrecorded path may be a pending delete, so every lazy fetch stays off.
	// It lifts once the table has room and the newest unrecorded delete is
	// past the TTL (skippedThroughTsNs): an unrecorded delete has no event
	// to confirm it, so only the TTL makes it safe, as for a pending one.
	full               bool
	skippedThroughTsNs int64
	// drainedOffsets records, per mount, the write-back offset the last
	// drain released through, so a read repeats the sweep only once the
	// offset has advanced.
	drainedOffsets map[string]int64
}

// isFull reports whether an unrecorded delete still keeps the lazy fetch off.
func (t *remoteDeletionTombstones) isFull() bool {
	if t == nil {
		return false
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.full
}

// drain, while the table is full, evicts expired tombstones and releases the
// mount's tombstones its write-back offset has passed, then re-evaluates
// the full state. It sweeps only when that offset advanced since the last
// drain for the mount or the unrecorded deletes have aged past the TTL, so
// a listing of many missing children does not rescan the table per child.
// haveOffset is false when the mount's persisted offset could not be read.
func (t *remoteDeletionTombstones) drain(mountDir string, nowNs, offsetTsNs int64, haveOffset bool) {
	if t == nil {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.drainedOffsets == nil {
		t.drainedOffsets = make(map[string]int64)
	}
	advanced := haveOffset && offsetTsNs > t.drainedOffsets[mountDir]
	aged := t.skippedThroughTsNs != 0 && nowNs-t.skippedThroughTsNs >= int64(remoteDeletionTombstoneTTL)
	if !advanced && !aged {
		return
	}
	t.evictExpiredLocked(nowNs)
	if advanced {
		t.releaseConsumedLocked(mountDir, offsetTsNs)
		t.drainedOffsets[mountDir] = offsetTsNs
	}
	t.observeLocked()
}

// releaseConsumedLocked drops the mount's tombstones whose event the mount's
// write-back offset has passed; a pending one has no event timestamp yet
// and stays, and another mount's tombstones answer to their own offset.
// Caller must hold t.mu.
func (t *remoteDeletionTombstones) releaseConsumedLocked(mountDir string, offsetTsNs int64) {
	prefix := strings.TrimSuffix(mountDir, "/") + "/"
	under := func(p string) bool { return p == mountDir || strings.HasPrefix(p, prefix) }
	for p, cur := range t.files {
		if under(p) && !t.pending[p] && cur.streamTsNs <= offsetTsNs {
			delete(t.files, p)
		}
	}
	for p, cur := range t.dirs {
		if under(p) && !t.pending[p] && cur.streamTsNs <= offsetTsNs {
			delete(t.dirs, p)
		}
	}
}

// observeLocked publishes the table size and lifts the full state once a
// delete fits again and the newest unrecorded delete is past the TTL.
// Caller must hold t.mu.
func (t *remoteDeletionTombstones) observeLocked() {
	size := len(t.files) + len(t.dirs)
	if size < remoteDeletionTombstoneLimit && (t.skippedThroughTsNs == 0 || time.Now().UnixNano()-t.skippedThroughTsNs >= int64(remoteDeletionTombstoneTTL)) {
		t.full = false
		t.skippedThroughTsNs = 0
	}
	stats.FilerRemoteDeletionTombstones.Set(float64(size))
	if t.full {
		stats.FilerRemoteDeletionTombstonesFull.Set(1)
	} else {
		stats.FilerRemoteDeletionTombstonesFull.Set(0)
	}
}

func newRemoteDeletionTombstones() *remoteDeletionTombstones {
	return &remoteDeletionTombstones{
		files:   make(map[string]remoteDeletionTombstone),
		dirs:    make(map[string]remoteDeletionTombstone),
		pending: make(map[string]bool),
	}
}

// add records a tombstone ahead of its metadata event — the timestamp is the
// local delete time, a lower bound the sync offset cannot order against.
func (t *remoteDeletionTombstones) add(path string, isDir bool, tsNs int64) {
	t.upsert(path, isDir, tsNs, 0, false)
}

// addFromEvent records a tombstone stamped by the delete event itself:
// commitTsNs orders it against other mutations, streamTsNs is the delivery
// position the mount's sync offset must pass to release it.
func (t *remoteDeletionTombstones) addFromEvent(path string, isDir bool, commitTsNs, streamTsNs int64) {
	t.upsert(path, isDir, commitTsNs, streamTsNs, true)
}

func (t *remoteDeletionTombstones) upsert(path string, isDir bool, commitTsNs, streamTsNs int64, fromEvent bool) {
	if t == nil {
		return
	}
	if streamTsNs == 0 {
		// No delivery position yet — the offset can never vouch for it.
		streamTsNs = math.MaxInt64
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	// An ancestor directory tombstone at least as new already covers the
	// path; recording it again only spends capacity.
	for p := path; ; {
		i := strings.LastIndexByte(p, '/')
		if i <= 0 {
			break
		}
		p = p[:i]
		if ancestor, ok := t.dirs[p]; ok && ancestor.commitTsNs >= commitTsNs {
			return
		}
	}
	m := t.files
	if isDir {
		m = t.dirs
	}
	if cur, ok := m[path]; ok {
		if commitTsNs < cur.commitTsNs {
			return
		}
		m[path] = remoteDeletionTombstone{commitTsNs: commitTsNs, streamTsNs: streamTsNs}
		if fromEvent {
			delete(t.pending, path)
		} else {
			// a newer local delete restamps the tombstone ahead of its
			// event — the offset cannot vouch for it until the event lands
			t.pending[path] = true
		}
		return
	}
	if len(t.files)+len(t.dirs) >= remoteDeletionTombstoneLimit {
		t.evictExpiredLocked(time.Now().UnixNano())
		if len(t.files)+len(t.dirs) >= remoteDeletionTombstoneLimit {
			if !t.full {
				glog.V(0).Infof("remote deletion tombstones full (%d), lazy remote fetch off until the table has room and the unrecorded deletes age out; first skipped %s", remoteDeletionTombstoneLimit, path)
			}
			t.full = true
			if commitTsNs > t.skippedThroughTsNs {
				t.skippedThroughTsNs = commitTsNs
			}
			t.observeLocked()
			return
		}
	}
	m[path] = remoteDeletionTombstone{commitTsNs: commitTsNs, streamTsNs: streamTsNs}
	if !fromEvent {
		t.pending[path] = true
	}
	if isDir {
		t.dropCoveredLocked(path, commitTsNs)
	}
	t.observeLocked()
}

// dropCoveredLocked removes descendant tombstones a new directory tombstone
// subsumes: their deletes predate it, so the ancestor already hides those
// remote objects. Descendants deleted later keep their own tombstone.
// Caller must hold t.mu.
func (t *remoteDeletionTombstones) dropCoveredLocked(dirPath string, tsNs int64) {
	prefix := dirPath + "/"
	for p, cur := range t.files {
		if cur.commitTsNs <= tsNs && strings.HasPrefix(p, prefix) {
			delete(t.files, p)
			delete(t.pending, p)
		}
	}
	for p, cur := range t.dirs {
		if cur.commitTsNs <= tsNs && strings.HasPrefix(p, prefix) {
			delete(t.dirs, p)
			delete(t.pending, p)
		}
	}
}

// drop removes the exact tombstone recorded for path, e.g. when the delete
// that recorded it fails before touching anything.
func (t *remoteDeletionTombstones) drop(path string, tsNs int64) {
	if t == nil {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if cur, ok := t.dirs[path]; ok && cur.commitTsNs <= tsNs {
		delete(t.dirs, path)
		delete(t.pending, path)
	}
	if cur, ok := t.files[path]; ok && cur.commitTsNs <= tsNs {
		delete(t.files, path)
		delete(t.pending, path)
	}
	t.observeLocked()
}

// clear drops a file tombstone when a write at the path is at least as new as
// the delete; a replayed older create must not lift a newer delete.
func (t *remoteDeletionTombstones) clear(path string, tsNs int64) {
	if t == nil {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if cur, ok := t.files[path]; ok && tsNs >= cur.commitTsNs {
		delete(t.files, path)
		delete(t.pending, path)
	}
	t.observeLocked()
}

// blockedSince returns the tombstone governing path — its own file tombstone
// or the newest one from a deleted ancestor directory — and whether that
// tombstone is still waiting for its event. commitTsNs == 0 means clear.
func (t *remoteDeletionTombstones) blockedSince(path string) (streamTsNs, commitTsNs int64, pending bool) {
	if t == nil {
		return 0, 0, false
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if cur, ok := t.files[path]; ok {
		streamTsNs, commitTsNs, pending = cur.streamTsNs, cur.commitTsNs, t.pending[path]
	}
	for p := path; ; {
		if cur, ok := t.dirs[p]; ok && cur.commitTsNs > commitTsNs {
			streamTsNs, commitTsNs, pending = cur.streamTsNs, cur.commitTsNs, t.pending[p]
		}
		i := strings.LastIndexByte(p, '/')
		if i <= 0 {
			break
		}
		p = p[:i]
	}
	return streamTsNs, commitTsNs, pending
}

// releaseThrough drops the tombstones governing path whose remote delete the
// write-back offset has passed, i.e. recorded at or below streamTsNs.
// Pending tombstones carry no delivery position yet and are never dropped.
func (t *remoteDeletionTombstones) releaseThrough(path string, tsNs int64) {
	if t == nil {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if cur, ok := t.files[path]; ok && cur.streamTsNs <= tsNs {
		delete(t.files, path)
		delete(t.pending, path)
	}
	for p := path; ; {
		if cur, ok := t.dirs[p]; ok && cur.streamTsNs <= tsNs {
			delete(t.dirs, p)
			delete(t.pending, p)
		}
		i := strings.LastIndexByte(p, '/')
		if i <= 0 {
			break
		}
		p = p[:i]
	}
	t.observeLocked()
}

// releaseExpired drops the tombstones governing path that outlived the TTL —
// the only release a tombstone still waiting for its event can get.
func (t *remoteDeletionTombstones) releaseExpired(path string, nowNs int64) {
	if t == nil {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if cur, ok := t.files[path]; ok && nowNs-cur.commitTsNs >= int64(remoteDeletionTombstoneTTL) {
		delete(t.files, path)
		delete(t.pending, path)
	}
	for p := path; ; {
		if cur, ok := t.dirs[p]; ok && nowNs-cur.commitTsNs >= int64(remoteDeletionTombstoneTTL) {
			delete(t.dirs, p)
			delete(t.pending, p)
		}
		i := strings.LastIndexByte(p, '/')
		if i <= 0 {
			break
		}
		p = p[:i]
	}
	t.observeLocked()
}

func (t *remoteDeletionTombstones) evictExpiredLocked(nowNs int64) {
	for p, cur := range t.files {
		if nowNs-cur.commitTsNs >= int64(remoteDeletionTombstoneTTL) {
			delete(t.files, p)
			delete(t.pending, p)
		}
	}
	for p, cur := range t.dirs {
		if nowNs-cur.commitTsNs >= int64(remoteDeletionTombstoneTTL) {
			delete(t.dirs, p)
			delete(t.pending, p)
		}
	}
	t.observeLocked()
}

// noteRemoteDeletion records a delete of path under a remote mount so lazy
// remote reads skip it until the remote delete is confirmed.
func (f *Filer) noteRemoteDeletion(p util.FullPath, isDir bool, tsNs int64) {
	if f.RemoteStorage == nil || f.remoteTombstones == nil {
		return
	}
	if _, remoteLoc := f.RemoteStorage.FindMountDirectory(p); remoteLoc == nil {
		return
	}
	f.remoteTombstones.add(string(p), isDir, tsNs)
}

// unnoteRemoteDeletion retracts a tombstone when the delete that recorded it
// fails before touching anything under path.
func (f *Filer) unnoteRemoteDeletion(p util.FullPath, tsNs int64) {
	if f.remoteTombstones == nil {
		return
	}
	f.remoteTombstones.drop(string(p), tsNs)
}

// isRemoteDeletionPending reports whether a remote write-back delete for p is
// still owed: p was deleted under mountDir and neither a rewrite, the mount's
// sync offset, nor the TTL has lifted the tombstone.
func (f *Filer) isRemoteDeletionPending(ctx context.Context, p util.FullPath, mountDir util.FullPath) bool {
	if f.remoteTombstones == nil {
		return false
	}
	// With deletes unrecorded, any path may be one: no lazy fetch resurrects
	// it until the table has room again and the unrecorded deletes aged out.
	// Reads keep the table draining meanwhile.
	if f.remoteTombstones.isFull() {
		offset, err := f.readRemoteSyncOffset(ctx, mountDir)
		f.remoteTombstones.drain(string(mountDir), time.Now().UnixNano(), offset, err == nil)
		if f.remoteTombstones.isFull() {
			return true
		}
	}
	streamTsNs, commitTsNs, pending := f.remoteTombstones.blockedSince(string(p))
	if commitTsNs == 0 {
		return false
	}
	if pending {
		// Recorded ahead of its delete event — the write-back offset cannot
		// vouch for it yet; only the TTL lifts it.
		if f.remoteDeletionExpired(commitTsNs) {
			f.remoteTombstones.releaseExpired(string(p), time.Now().UnixNano())
			return false
		}
		return true
	}
	if f.remoteDeletionConsumed(ctx, mountDir, streamTsNs) {
		f.remoteTombstones.releaseThrough(string(p), streamTsNs)
		return false
	}
	return true
}

func (f *Filer) remoteDeletionExpired(tsNs int64) bool {
	return time.Now().UnixNano()-tsNs >= int64(remoteDeletionTombstoneTTL)
}

func (f *Filer) remoteDeletionConsumed(ctx context.Context, mountDir util.FullPath, tsNs int64) bool {
	if f.remoteDeletionExpired(tsNs) {
		return true
	}
	offset, err := f.readRemoteSyncOffset(ctx, mountDir)
	return err == nil && offset >= tsNs
}

// readRemoteSyncOffset reads the write-back daemon's persisted watermark for
// mountDir straight from the local store: every event at or below it has been
// applied to the remote.
func (f *Filer) readRemoteSyncOffset(ctx context.Context, mountDir util.FullPath) (int64, error) {
	value, err := f.Store.KvGet(ctx, remote_storage.SyncOffsetKey(string(mountDir)))
	if err != nil {
		return 0, err
	}
	if len(value) < 8 {
		return 0, nil
	}
	return int64(util.BytesToUint64(value)), nil
}

// RebuildRemoteDeletionTombstones gates lazy remote reads and replays the
// persisted metadata log from the oldest write-back offset across mounts,
// restoring tombstones for deletes committed before a restart but not yet
// applied to the remote. Every filer writes its log files under the same
// directory, so pending peer deletes replay too; only events still inside
// the unflushed buffer tail are missed. The gate is set synchronously so no
// lazy read can slip in before replay starts, and it stays closed until a
// replay succeeds.
func (f *Filer) RebuildRemoteDeletionTombstones(ctx context.Context) {
	if f.RemoteStorage == nil || f.remoteTombstones == nil {
		return
	}
	mounts := f.RemoteStorage.MountedDirectories()
	if len(mounts) == 0 {
		return
	}
	done := make(chan struct{})
	f.remoteTombstonesDone.Store(&done)
	go f.rebuildRemoteDeletionTombstones(ctx, mounts, done)
}

func (f *Filer) rebuildRemoteDeletionTombstones(ctx context.Context, mounts []util.FullPath, done chan struct{}) {
	// the replay itself lists directories; do not let it wait on its own gate
	ctx = context.WithValue(ctx, lazyFetchContextKey{}, true)
	startTsNs := f.remoteDeletionRebuildStartTsNs(ctx, mounts)
	openGate := func() {
		close(done)
		f.remoteTombstonesDone.Store(nil)
	}
	// Deletes made after startup are recorded through the live delete and
	// event paths, so replay can only be missing deletes committed before
	// the restart — and those have all crossed the tombstone TTL once this
	// deadline passes. Holding the gate longer protects nothing.
	replayDeadline := time.Now().Add(remoteDeletionTombstoneTTL)
	backoff := 2 * time.Second
	for {
		_, _, err := f.ReadPersistedLogBuffer(ctx, log_buffer.NewMessagePosition(startTsNs, 0), 0,
			func(logEntry *filer_pb.LogEntry) (bool, error) {
				event := &filer_pb.SubscribeMetadataResponse{}
				if err := proto.Unmarshal(logEntry.Data, event); err != nil {
					return false, nil
				}
				// Same normalization as delivery: the payload ts is the
				// commit stamp, the wrapper ts the stream position.
				if event.MutationTsNs == 0 {
					event.MutationTsNs = event.TsNs
				}
				event.TsNs = logEntry.TsNs
				f.onRemoteDeletionEvents(event)
				return false, nil
			})
		if err == nil {
			openGate()
			return
		}
		glog.WarningfCtx(ctx, "rebuild remote deletion tombstones: %v", err)
		if !time.Now().Before(replayDeadline) {
			glog.ErrorfCtx(ctx, "rebuild remote deletion tombstones: giving up after %v", remoteDeletionTombstoneTTL)
			openGate()
			return
		}
		select {
		case <-ctx.Done():
			return
		case <-time.After(backoff):
		}
		if backoff < time.Minute {
			backoff *= 2
		}
	}
}

// remoteDeletionRebuildStartTsNs returns the oldest write-back offset across
// mounts — the earliest event the daemon may not have applied. Mounts without
// a recorded offset replay from the TTL floor: events older than it would
// build tombstones that are already expired.
func (f *Filer) remoteDeletionRebuildStartTsNs(ctx context.Context, mounts []util.FullPath) int64 {
	startTsNs := int64(math.MaxInt64)
	for _, dir := range mounts {
		offset, err := f.readRemoteSyncOffset(ctx, dir)
		if err != nil {
			glog.WarningfCtx(ctx, "read remote sync offset for %s: %v", dir, err)
			offset = 0
		}
		if offset < startTsNs {
			startTsNs = offset
		}
	}
	ttlFloor := time.Now().Add(-remoteDeletionTombstoneTTL).UnixNano()
	if startTsNs == int64(math.MaxInt64) || startTsNs < ttlFloor {
		return ttlFloor
	}
	return startTsNs
}

// onRemoteDeletionEvents folds peer and local metadata events into the
// tombstone set: a delete or rename source is tombstoned at the event
// timestamp, a create/update/rename target lifts a file tombstone.
func (f *Filer) onRemoteDeletionEvents(event *filer_pb.SubscribeMetadataResponse) {
	message := event.EventNotification
	if message == nil {
		return
	}
	// The commit stamp orders the delete against other mutations; event.TsNs
	// is the delivery position the write-back offset must pass.
	commitTsNs := eventMutationTsNs(event)
	if message.OldEntry != nil && !IsMetadataOnlyDelete(message.OldEntry) {
		sourcePath := filer_pb.MetadataEventSourceFullPath(event)
		if message.NewEntry == nil || sourcePath != filer_pb.MetadataEventTargetFullPath(event) {
			if f.RemoteStorage != nil && f.remoteTombstones != nil {
				if _, remoteLoc := f.RemoteStorage.FindMountDirectory(util.FullPath(sourcePath)); remoteLoc != nil {
					f.remoteTombstones.addFromEvent(sourcePath, message.OldEntry.IsDirectory, commitTsNs, event.TsNs)
				}
			}
		}
	}
	if message.NewEntry != nil {
		f.remoteTombstones.clear(filer_pb.MetadataEventTargetFullPath(event), commitTsNs)
	}
}