Compare commits

...
Author SHA1 Message Date
chrislusf bc97217591 filer: reflow the replay-lock comment 2026-10-25 14:10:06 -07:00
chrislusf 7664bb3b7f filer: guard the record read against a nil store 2026-10-25 13:57:18 -07:00
chrislusf 4ae402de0d filer: cover retained commit records fencing replayed events 2026-10-25 13:50:39 -07:00
chrislusf ef72d3976e filer: keep reading per-path commit records older builds left
4.49 and 4.50 wrote an MpTs record for every metadata change and an MpDl
record for every delete, kept after the entry's removal. The entry-stamp
scheme replaced the writers, but the merge also dropped the readers, so a
store upgraded from a recording build loses the ordering those records
still carry: a peer replaying an event older than a recorded change or
delete can resurrect the path or revert a newer entry.

Restore the read side only — metaPathKey, kvReadTsNs, and the ancestor
walk for delete records, all unchanged from the writing builds. No new
records are written, so per-mutation overhead stays zero; the lookups run
only on the replay and traversal paths, which dedicated-store peers are
the only filers to reach.
2026-10-25 13:50:36 -07:00
3 changed files with 236 additions and 27 deletions

No files matched your search

+119 -12
View File
@@ -2,12 +2,56 @@ package filer
import (
"context"
"crypto/sha256"
"errors"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
// metaPathTsPrefix keys the per-path commit timestamps older builds
// wrote into the store KV for every mutation and kept after the entry
// was deleted. They are no longer written — ordering lives on the
// entry's CommitTsNs — but records left behind by an upgrade still
// fence replayed events older than the change they recorded.
metaPathTsPrefix = "MpTs"
// metaPathDelPrefix keys the delete timestamps those builds kept. The
// replay check consults the path's ancestors too, so a directory's
// retained delete record covers children deleted without per-child
// events.
metaPathDelPrefix = "MpDl"
)
// metaPathKey hashes the path into the fixed-size key the writing builds
// used: SQL stores split the key at byte 8 and base64 the rest into a
// bounded name column, which a raw path can overflow.
func metaPathKey(prefix string, path util.FullPath) []byte {
sum := sha256.Sum256([]byte(path))
key := make([]byte, len(prefix)+16)
copy(key, prefix)
copy(key[len(prefix):], sum[:16])
return key
}
// kvReadTsNs returns the timestamp recorded for key, or found=false when the
// key is absent or malformed. Any other store read failure is an error: a
// caller deciding whether a replayed change is stale must not guess.
func (f *Filer) kvReadTsNs(ctx context.Context, key []byte) (tsNs int64, found bool, err error) {
value, err := f.Store.KvGet(ctx, key)
if err != nil {
if errors.Is(err, ErrKvNotFound) {
return 0, false, nil
}
return 0, false, err
}
if len(value) < 8 {
return 0, false, nil
}
return int64(util.BytesToUint64(value)), true, nil
}
// CommitTsNs resolves the commit timestamp at path for traversal: the
// entry's stamp, else fallbackTsNs for pre-stamp or missing entries.
func (f *Filer) CommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
@@ -17,40 +61,103 @@ func (f *Filer) CommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs
entry, err := f.Store.FindEntry(ctx, path)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return fallbackTsNs, nil
return f.retainedCommitTsNs(ctx, path, fallbackTsNs)
}
return 0, err
}
if entry == nil {
return fallbackTsNs, nil
return f.retainedCommitTsNs(ctx, path, fallbackTsNs)
}
return f.EntryCommitTsNs(ctx, entry, fallbackTsNs)
}
// EntryCommitTsNs returns the stamp on an already-loaded entry, else
// fallbackTsNs.
// EntryCommitTsNs returns the newest commit time known for an already-loaded
// entry: its stamp, a retained per-path record left by an older build —
// which can outdate a replayed peer stamp — else fallbackTsNs.
func (f *Filer) EntryCommitTsNs(ctx context.Context, entry *Entry, fallbackTsNs int64) (int64, error) {
if entry.CommitTsNs != 0 {
return entry.CommitTsNs, nil
tsNs := entry.CommitTsNs
if f.Store == nil {
if tsNs == 0 {
return fallbackTsNs, nil
}
return tsNs, nil
}
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, entry.FullPath))
if err != nil {
return 0, err
}
if found && recTs > tsNs {
tsNs = recTs
}
if tsNs == 0 {
return fallbackTsNs, nil
}
return tsNs, nil
}
// retainedCommitTsNs reads the per-path record an older build may have left
// for a path that has no entry now — a delete under the recording builds
// kept its change record past the entry's removal.
func (f *Filer) retainedCommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
if err != nil {
return 0, err
}
if found {
return recTs, nil
}
return fallbackTsNs, nil
}
// deletedAfterTsNs returns the newest retained delete record among path and
// its ancestors.
func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64, error) {
var tsNs int64
for p := string(path); p != "" && p != "/"; {
ts, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathDelPrefix, util.FullPath(p)))
if err != nil {
return 0, err
}
if found && ts > tsNs {
tsNs = ts
}
i := strings.LastIndexByte(p, '/')
if i <= 0 {
break
}
p = p[:i]
}
return tsNs, nil
}
// localCommitTsNs returns this filer's newest known commit time for path:
// the entry's stamp — not client-controllable like mtime — else mtime.
// the entry's stamp — not client-controllable like mtime — else mtime, plus
// any retained change or delete record an older build left for the path or
// its ancestors.
func (f *Filer) localCommitTsNs(ctx context.Context, path util.FullPath) (int64, error) {
if f.Store == nil {
return 0, nil
}
var tsNs int64
entry, err := f.Store.FindEntry(ctx, path)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return 0, nil
if !errors.Is(err, filer_pb.ErrNotFound) {
return 0, err
}
return 0, err
}
if entry == nil {
return 0, nil
if tsNs, err = f.retainedCommitTsNs(ctx, path, 0); err != nil {
return 0, err
}
} else if tsNs, err = f.EntryCommitTsNs(ctx, entry, entry.Mtime.UnixNano()); err != nil {
return 0, err
}
return f.EntryCommitTsNs(ctx, entry, entry.Mtime.UnixNano())
delTs, err := f.deletedAfterTsNs(ctx, path)
if err != nil {
return 0, err
}
if delTs > tsNs {
tsNs = delTs
}
return tsNs, nil
}
+100
View File
@@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
)
@@ -89,3 +90,102 @@ func TestEventWithoutPreStampStillEmits(t *testing.T) {
t.Fatalf("stamp leaked: %d", got)
}
}
// putRecord seeds the per-path key an older build would have left.
func putRecord(t *testing.T, store *stubFilerStore, prefix string, path util.FullPath, tsNs int64) {
t.Helper()
value := make([]byte, 8)
util.Uint64toBytes(value, uint64(tsNs))
if err := store.KvPut(context.Background(), metaPathKey(prefix, path), value); err != nil {
t.Fatalf("seed record %s%s: %v", prefix, path, err)
}
}
// A retained change record still orders a path whose entry is gone: a
// replayed create older than the record must not resurrect it.
func TestRetainedChangeRecordFencesMissingEntry(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
ctx := context.Background()
putRecord(t, store, metaPathTsPrefix, "/dir/file", 300)
if newer, err := f.replayBlockedByLocal(ctx, "/dir/file", 200); err != nil || !newer {
t.Fatalf("event at 200 should lose to retained record at 300: newer=%v err=%v", newer, err)
}
if newer, err := f.replayBlockedByLocal(ctx, "/dir/file", 400); err != nil || newer {
t.Fatalf("event at 400 should win over retained record at 300: newer=%v err=%v", newer, err)
}
}
// A directory's retained delete record covers children removed without
// per-child records — the replay check walks the ancestors.
func TestRetainedAncestorDeleteRecordFencesChild(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
ctx := context.Background()
putRecord(t, store, metaPathDelPrefix, "/dir", 300)
if newer, err := f.replayBlockedByLocal(ctx, "/dir/sub/file", 200); err != nil || !newer {
t.Fatalf("child event at 200 should lose to ancestor delete at 300: newer=%v err=%v", newer, err)
}
if newer, err := f.replayBlockedByLocal(ctx, "/dir/sub/file", 400); err != nil || newer {
t.Fatalf("child event at 400 should win over ancestor delete at 300: newer=%v err=%v", newer, err)
}
}
// The retained record orders the entry, not its client-settable mtime: a
// mounted client can push mtime into the future, and the record must not
// let that silence newer replayed events.
func TestRetainedRecordOrdersUnstampedEntry(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
ctx := context.Background()
path := util.FullPath("/dir/file")
putRecord(t, store, metaPathTsPrefix, path, 300)
store.entries[string(path)] = &Entry{
FullPath: path,
Attr: Attr{Mtime: time.Unix(0, 999)},
}
if tsNs, err := f.localCommitTsNs(ctx, path); err != nil || tsNs != 300 {
t.Fatalf("localCommitTsNs = %d,%v; want the record's 300, not the entry's mtime", tsNs, err)
}
}
// An entry stamped under the current build outranks any record an older
// build left: the stamp is the newest commit.
func TestEntryStampBeatsRetainedRecord(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
ctx := context.Background()
path := util.FullPath("/dir/file")
putRecord(t, store, metaPathTsPrefix, path, 300)
store.entries[string(path)] = &Entry{FullPath: path, CommitTsNs: 400}
if tsNs, err := f.localCommitTsNs(ctx, path); err != nil || tsNs != 400 {
t.Fatalf("localCommitTsNs = %d,%v; want the entry stamp's 400", tsNs, err)
}
}
// End to end: a replayed create older than a retained delete record leaves
// the path absent.
func TestReplayCreateBlockedByRetainedDelete(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
ctx := context.Background()
putRecord(t, store, metaPathDelPrefix, "/dir/file", 300)
event := &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("resurrected")},
},
TsNs: 200,
}
if err := f.Replay(event); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(ctx, "/dir/file"); err == nil {
t.Fatal("replayed create at 200 must not resurrect a path deleted at 300")
}
}
+17 -15
View File
@@ -23,12 +23,14 @@ import (
// only when no newer local change is recorded, so a late event cannot revert
// or resurrect entries.
//
// Ordering uses the per-path commit timestamp recorded for every emitted
// mutation (meta_path_ts.go), which survives deletes as a tombstone and is
// not client-controllable the way entry mtime is — a mounted client can set
// mtime into the future, and a rename keeps the source file's mtime on the
// destination entry. Entries written before recording existed fall back to
// their mtime.
// Ordering uses the commit timestamp stamped on the entry by its mutation
// event (meta_path_ts.go), which is not client-controllable the way entry
// mtime is — a mounted client can set mtime into the future, and a rename
// keeps the source file's mtime on the destination entry. Entries written
// before stamping existed fall back to their mtime, except where a
// per-path record left by an older build remains: a retained change or
// delete record outlives the entry it ordered, so a stale replayed event
// cannot resurrect a path whose removal the record still covers.
//
// The check and the mutation run under the same per-path entry lock the
// filer server's local writes hold, so they cannot be split by a concurrent
@@ -340,19 +342,19 @@ func (f *Filer) replayBlockedByLocal(ctx context.Context, path util.FullPath, ts
return localTsNs > tsNs, nil
}
// lockReplayPaths holds the shared entry locks for every path the event
// touches, in the same sorted order the rename handler takes them.
// lockReplayPaths takes the per-path entry locks for the event's paths and
// every ancestor of them, in lexicographic order — parents sort before
// children, the same ordering the local rename handler's two-path lock uses,
// so lock acquisition is deadlock-free across operations. Ancestors are
// included because the replay check consults ancestor delete records: a
// local recursive delete or directory rename holds only the ancestor's lock,
// and without contending on it a replay could reinsert a stale child after
// its check. Ancestors are locked shared — they only need to exclude an
// exclusive holder, not each other, or every replay under a busy parent
// would serialize — while a path the event actually changes stays exclusive
// even when it is also the other path's ancestor. The root is never locked:
// included because the replay check consults ancestor state — a retained
// delete record an older build left, or the ancestor entry itself: a local
// recursive delete or directory rename holds only the ancestor's lock, and
// without contending on it a replay could reinsert a stale child into a
// subtree being deleted after its check. Ancestors are locked shared —
// they only need to exclude an exclusive holder, not each other, or every
// replay under a busy parent would serialize — while a path the event
// actually changes stays exclusive even when it is also the other path's
// ancestor. The root is never locked:
// it cannot be deleted, and a shared root lock would still be excluded by
// every top-level delete.
func (f *Filer) lockReplayPaths(oldPath, newPath util.FullPath) func() {