seaweedfs/weed/filer/meta_path_ts.go

package filer

import (
	"context"
	"crypto/sha256"
	"errors"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

const (
	// metaPathTsPrefix keys per-path commit timestamps in the filer store KV:
	// the newest event timestamp this filer committed or applied for the
	// path. Written for every emitted mutation event and kept after the entry
	// is deleted, so a replayed peer event can still be ordered against a
	// change that left no entry behind.
	metaPathTsPrefix = "MpTs"
	// metaPathDelPrefix keys delete-only timestamps. A replayed event checks
	// the path's ancestors too, so a directory delete also covers children
	// swept without per-child events (a dropped bucket's whole-store delete,
	// or a walk aborted partway).
	metaPathDelPrefix = "MpDl"
)

// metaPathKey hashes the path into a fixed-size key. SQL-backed KV stores
// split the key at byte 8 and store the remainder base64-encoded in a
// VARCHAR(766) name column: a raw path over ~578 bytes overflows it, the
// KvPut insert fails, and the fallback update touches no row yet returns no
// error — the commit record would never land and stale replay could restore
// the path.
func metaPathKey(prefix string, path util.FullPath) []byte {
	sum := sha256.Sum256([]byte(path))
	key := make([]byte, len(prefix)+16)
	copy(key, prefix)
	copy(key[len(prefix):], sum[:16])
	return key
}

// kvReadTsNs returns the timestamp recorded for key, or found=false when the
// key is absent or malformed. Any other store read failure is an error: a
// caller deciding whether a replayed change is stale must not guess.
func (f *Filer) kvReadTsNs(ctx context.Context, key []byte) (tsNs int64, found bool, err error) {
	value, err := f.Store.KvGet(ctx, key)
	if err != nil {
		if errors.Is(err, ErrKvNotFound) {
			return 0, false, nil
		}
		return 0, false, err
	}
	if len(value) < 8 {
		return 0, false, nil
	}
	return int64(util.BytesToUint64(value)), true, nil
}

// notePathChange persists tsNs as path's newest committed change. Local event
// timestamps are stamped monotonically (metaLogInflight.stamp), so a plain
// put cannot regress an earlier local record; a higher peer record it may
// overwrite was already newer under last-writer-wins ordering anyway. A
// delete additionally writes a delete record the ancestor lookup consults.
//
// The write uses a fresh context: callers notify after CommitTransaction,
// and SQL stores keep the already-committed *sql.Tx in the request context —
// a record written through it would fail and the path would lose ordering.
//
// A failed write is kept in metaTsPending and retried in the background: the
// mutation the record orders has already committed, so dropping it would let
// a stale replayed change overwrite or resurrect the path while the store is
// unreachable. Ordering reads consult the pending record until the retry
// lands it.
func (f *Filer) notePathChange(path util.FullPath, tsNs int64, deleted bool) {
	if f.Store == nil {
		return
	}
	ctx := context.Background()
	key := string(path)
	rec := f.pendingRec(key)
	defer rec.mu.Unlock()

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(tsNs))
	if err := f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); err != nil {
		glog.WarningfCtx(ctx, "record meta change time %s: %v", path, err)
		if tsNs > rec.changeTsNs.Load() {
			rec.changeTsNs.Store(tsNs)
		}
	} else if rec.changeTsNs.Load() <= tsNs {
		// This write covers every change record owed so far.
		rec.changeTsNs.Store(0)
	}
	if deleted {
		if err := f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); err != nil {
			glog.WarningfCtx(ctx, "record meta delete time %s: %v", path, err)
			if tsNs > rec.deleteTsNs.Load() {
				rec.deleteTsNs.Store(tsNs)
			}
		} else if rec.deleteTsNs.Load() <= tsNs {
			rec.deleteTsNs.Store(0)
		}
	}
	if rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0 {
		f.metaTsPending.CompareAndDelete(key, rec)
		return
	}
	if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
		go f.repairMetaPathTs(path)
	}
}

// metaTsPendingRec is a commit record still on its way to the store, holding
// only work not yet written: changeTsNs is the newest committed change the
// MpTs record lacks and deleteTsNs the newest delete the MpDl record lacks —
// the two keep their own times so a later update cannot masquerade as the
// delete. Store writes and the bookkeeping they discharge share mu, so a
// repair pass can never write an older record over a newer one that already
// landed; readers take the atomic fields directly.
type metaTsPendingRec struct {
	mu         sync.Mutex
	changeTsNs atomic.Int64
	deleteTsNs atomic.Int64
}

// pendingRec returns the live pending record for key, locked for mutation,
// recreating it when a repair already dropped the previous one.
func (f *Filer) pendingRec(key string) *metaTsPendingRec {
	for {
		cur, _ := f.metaTsPending.LoadOrStore(key, &metaTsPendingRec{})
		rec := cur.(*metaTsPendingRec)
		rec.mu.Lock()
		if live, ok := f.metaTsPending.Load(key); ok && live == cur {
			return rec
		}
		rec.mu.Unlock()
	}
}

// repairMetaPathTs retries a pending commit record until it lands. Each pass
// writes only what the record still owes, so a merge by a later failure is
// picked up without another goroutine and a landed write is never rewritten.
// The retry is unbounded — the mutation is already committed and only the
// record preserves its ordering.
func (f *Filer) repairMetaPathTs(path util.FullPath) {
	key := string(path)
	ctx := f.flushCtx
	if ctx == nil {
		ctx = context.Background()
	}
	defer func() {
		f.metaTsRepairing.Delete(key)
		// A failure landing between the last pass and the flag release
		// still needs a repair pass — unless the retry context is done.
		if ctx.Err() != nil {
			return
		}
		if _, ok := f.metaTsPending.Load(key); ok {
			if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
				go f.repairMetaPathTs(path)
			}
		}
	}()
	for {
		if _, ok := f.metaTsPending.Load(key); !ok {
			return
		}
		rec := f.pendingRec(key)
		var tsErr, delErr error
		if tsNs := rec.changeTsNs.Load(); tsNs > 0 {
			value := make([]byte, 8)
			util.Uint64toBytes(value, uint64(tsNs))
			if tsErr = f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); tsErr == nil {
				rec.changeTsNs.Store(0)
			}
		}
		if tsNs := rec.deleteTsNs.Load(); tsNs > 0 {
			value := make([]byte, 8)
			util.Uint64toBytes(value, uint64(tsNs))
			if delErr = f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); delErr == nil {
				rec.deleteTsNs.Store(0)
			}
		}
		done := rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0
		if done {
			f.metaTsPending.CompareAndDelete(key, rec)
		}
		rec.mu.Unlock()
		if !done {
			glog.V(0).InfofCtx(ctx, "retry record meta change time %s: %v %v", path, tsErr, delErr)
			select {
			case <-ctx.Done():
				return
			case <-time.After(time.Second):
			}
		}
	}
}

// pendingPathTsNs returns the newest pending commit timestamp for path and
// the newest pending delete timestamp.
func (f *Filer) pendingPathTsNs(path util.FullPath) (changeTsNs, deleteTsNs int64) {
	cur, ok := f.metaTsPending.Load(string(path))
	if !ok {
		return 0, 0
	}
	rec := cur.(*metaTsPendingRec)
	return rec.changeTsNs.Load(), rec.deleteTsNs.Load()
}

// CommitTsNs returns the newest commit timestamp known for path — the
// recorded event time, or a still-pending one — and fallbackTsNs when none
// exists. Peers stamp traversal entries with it so a bootstrap subscriber
// orders each entry on the sender's commit clock rather than the entry's
// client-settable mtime. A store read failure is an error, not a fallback:
// the mtime it would mint could outdate real commits.
func (f *Filer) CommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
	if f.Store == nil {
		return fallbackTsNs, nil
	}
	tsNs := fallbackTsNs
	recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
	if err != nil {
		return 0, err
	}
	if found {
		tsNs = recTs
	}
	if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
		tsNs = pendTs
	}
	return tsNs, nil
}

// noteCommittedEvent records the event's commit timestamp for every path it
// touches: the source of a delete or rename, and the target of a create,
// update, or rename. It runs after the local mutation commits while the
// caller still holds the path's entry lock (for filer-server mutations), so
// a peer replay ordered by the same lock sees either the whole local change
// or none of it.
func (f *Filer) noteCommittedEvent(event *filer_pb.SubscribeMetadataResponse) {
	message := event.EventNotification
	if message == nil {
		return
	}
	var source, target util.FullPath
	if message.OldEntry != nil {
		source = util.FullPath(filer_pb.MetadataEventSourceFullPath(event))
	}
	if message.NewEntry != nil {
		target = util.FullPath(filer_pb.MetadataEventTargetFullPath(event))
	}
	// A delete or a rename removes the source path; an update reports the
	// same path on both sides and counts once.
	if source != "" && source != target {
		f.notePathChange(source, event.TsNs, true)
	}
	if target != "" {
		f.notePathChange(target, event.TsNs, false)
	}
}

// deletedAfterTsNs returns the newest recorded delete timestamp among path
// and its ancestors.
func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64, error) {
	var tsNs int64
	for p := string(path); p != "" && p != "/"; {
		ts, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathDelPrefix, util.FullPath(p)))
		if err != nil {
			return 0, err
		}
		if found && ts > tsNs {
			tsNs = ts
		}
		if _, pendDel := f.pendingPathTsNs(util.FullPath(p)); pendDel > tsNs {
			tsNs = pendDel
		}
		i := strings.LastIndexByte(p, '/')
		if i <= 0 {
			break
		}
		p = p[:i]
	}
	return tsNs, nil
}

// localCommitTsNs returns the newest timestamp this filer is known to have
// committed for path: the recorded event time when one exists — it is not
// client-controllable the way an entry's mtime is — else the entry's mtime
// for entries written before recording existed, plus any recorded delete of
// the path or an ancestor.
func (f *Filer) localCommitTsNs(ctx context.Context, path util.FullPath) (int64, error) {
	if f.Store == nil {
		return 0, nil
	}
	var tsNs int64
	recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
	if err != nil {
		return 0, err
	}
	if found {
		tsNs = recTs
	} else if entry, findErr := f.Store.FindEntry(ctx, path); findErr != nil {
		if !errors.Is(findErr, filer_pb.ErrNotFound) {
			return 0, findErr
		}
	} else if entry != nil {
		tsNs = entry.Mtime.UnixNano()
	}
	if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
		tsNs = pendTs
	}
	delTs, err := f.deletedAfterTsNs(ctx, path)
	if err != nil {
		return 0, err
	}
	if delTs > tsNs {
		tsNs = delTs
	}
	return tsNs, nil
}