diff --git a/weed/server/filer_grpc_server_dlm.go b/weed/server/filer_grpc_server_dlm.go index b6516dcf54..7cdc696eb6 100644 --- a/weed/server/filer_grpc_server_dlm.go +++ b/weed/server/filer_grpc_server_dlm.go @@ -3,6 +3,7 @@ package weed_server import ( "context" "fmt" + "slices" "strings" "time" @@ -178,6 +179,9 @@ func (fs *FilerServer) ReplicateLock(ctx context.Context, req *filer_pb.Replicat // 2. Re-replicating to the new backup // 3. Transferring locks that no longer belong here func (fs *FilerServer) OnDlmChangeSnapshot(snapshot []pb.ServerAddress) { + if slices.Contains(snapshot, fs.option.Host) { + fs.resetLockRingClock() + } locks := fs.filer.Dlm.AllLocks() if len(locks) == 0 { return diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 5bbbc733a1..f30cab70f7 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -120,6 +120,10 @@ type FilerOption struct { type FilerServer struct { inFlightDataSize int64 inFlightUploads int64 + // unix nanos when this filer was first seen missing from its own lock ring; + // <= 0 while it is in the ring, with a new value on every reset + lockRingUnhealthySince atomic.Int64 + lockRingResets atomic.Int64 inFlightDataLimitCond *sync.Cond diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 3db3f13f29..bedf8c6595 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "path" + "slices" "strconv" "strings" "sync/atomic" @@ -390,12 +391,68 @@ func pathHasComponentPrefix(reqPath, prefix string) bool { return strings.HasPrefix(cleanedPath, cleanedPrefix+"/") } +// lockRingUnhealthyGracePeriod covers the first ring update after announcing +// and the master's ring-change batching delay. +const lockRingUnhealthyGracePeriod = 30 * time.Second + +// lockRingHealthy reports false once this filer has been missing from its own +// lock ring past the grace period: locked writes cannot be routed to it, and +// a restart re-registers it with the master. +func (fs *FilerServer) lockRingHealthy() bool { + dlm := fs.filer.Dlm + if dlm == nil || dlm.LockRing == nil || dlm.LockRing.Ring == nil { + return true + } + if announce := fs.option.AnnounceCh; announce != nil { + select { + case <-announce: + default: + return true + } + } + absent := fs.lockRingAbsence(func() bool { return slices.Contains(dlm.LockRing.Ring.GetServers(), dlm.Host) }) + if absent < lockRingUnhealthyGracePeriod { + return true + } + glog.Warningf("filerHealthz: %s absent from lock ring for %v (ring: %v)", + dlm.Host, absent.Round(time.Second), dlm.LockRing.Ring.GetServers()) + return false +} + +// lockRingAbsence returns how long inRing has reported this filer missing, +// or 0 while it is in the ring. +func (fs *FilerServer) lockRingAbsence(inRing func() bool) time.Duration { + since := fs.lockRingUnhealthySince.Load() + if inRing() { + fs.resetLockRingClock() + return 0 + } + // A rejoin after the load has stored a new reset value, so the clock + // neither starts nor is read across it. + if since <= 0 { + fs.lockRingUnhealthySince.CompareAndSwap(since, time.Now().UnixNano()) + return 0 + } + if fs.lockRingUnhealthySince.Load() != since { + return 0 + } + return time.Since(time.Unix(0, since)) +} + +func (fs *FilerServer) resetLockRingClock() { + fs.lockRingUnhealthySince.Store(-fs.lockRingResets.Add(1)) +} + func (fs *FilerServer) filerHealthzHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Server", "SeaweedFS "+version.VERSION) if _, err := fs.filer.Store.FindEntry(context.Background(), filer.TopicsDir); err != nil && err != filer_pb.ErrNotFound { glog.Warningf("filerHealthzHandler FindEntry: %+v", err) w.WriteHeader(http.StatusServiceUnavailable) - } else { - w.WriteHeader(http.StatusOK) + return } + if !fs.lockRingHealthy() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) } diff --git a/weed/server/filer_server_lock_ring_health_test.go b/weed/server/filer_server_lock_ring_health_test.go new file mode 100644 index 0000000000..ba1094640b --- /dev/null +++ b/weed/server/filer_server_lock_ring_health_test.go @@ -0,0 +1,155 @@ +package weed_server + +import ( + "slices" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb" +) + +func TestLockRingHealthy(t *testing.T) { + self := pb.ServerAddress("filer-0:8888") + dlm := lock_manager.NewDistributedLockManager(self) + fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{}} + + // Empty ring, within grace period: healthy. + if !fs.lockRingHealthy() { + t.Fatal("expected healthy during grace period") + } + + // Empty ring past the grace period: unhealthy. + fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano()) + if fs.lockRingHealthy() { + t.Fatal("expected unhealthy after grace period with empty ring") + } + + // Ring without self past the grace period: still unhealthy. + dlm.LockRing.SetSnapshot([]pb.ServerAddress{"filer-1:8888"}, 1) + if fs.lockRingHealthy() { + t.Fatal("expected unhealthy when self is not in ring") + } + + // Self back in ring: healthy and timer reset. + dlm.LockRing.SetSnapshot([]pb.ServerAddress{self}, 2) + if !fs.lockRingHealthy() { + t.Fatal("expected healthy when self is in ring") + } + if fs.lockRingUnhealthySince.Load() > 0 { + t.Fatal("expected unhealthy timer to reset") + } +} + +func TestLockRingHealthyWaitsForAnnounce(t *testing.T) { + dlm := lock_manager.NewDistributedLockManager("filer-0:8888") + announce := make(chan struct{}) + fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{AnnounceCh: announce}} + expired := time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano() + + if !fs.lockRingHealthy() || fs.lockRingUnhealthySince.Load() > 0 { + t.Fatal("expected healthy with no absence clock while registration is held") + } + fs.lockRingUnhealthySince.Store(expired) + if !fs.lockRingHealthy() { + t.Fatal("expected healthy while registration is held") + } + + close(announce) + fs.lockRingUnhealthySince.Store(0) + if !fs.lockRingHealthy() || fs.lockRingUnhealthySince.Load() <= 0 { + t.Fatal("expected the first probe after announcing to start the grace period") + } + fs.lockRingUnhealthySince.Store(expired) + if fs.lockRingHealthy() { + t.Fatal("expected unhealthy after announcing with an empty ring") + } +} + +func TestLockRingHealthyRestartsGraceAfterUnprobedRejoin(t *testing.T) { + self := pb.ServerAddress("filer-0:8888") + dlm := lock_manager.NewDistributedLockManager(self) + fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{Host: self}} + dlm.LockRing.SetTakeSnapshotCallback(fs.OnDlmChangeSnapshot) + + fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano()) + dlm.LockRing.SetSnapshot([]pb.ServerAddress{self, "filer-1:8888"}, 1) + dlm.LockRing.SetSnapshot([]pb.ServerAddress{"filer-1:8888"}, 2) + if !fs.lockRingHealthy() { + t.Fatal("expected a fresh grace period after rejoining the ring between probes") + } +} + +func TestLockRingHealthyDoesNotWaitOnRingMutex(t *testing.T) { + dlm := lock_manager.NewDistributedLockManager("filer-0:8888") + fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{}} + dlm.LockRing.SetSnapshot([]pb.ServerAddress{"filer-1:8888"}, 1) + fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano()) + + dlm.LockRing.Lock() + defer dlm.LockRing.Unlock() + done := make(chan bool, 1) + go func() { done <- fs.lockRingHealthy() }() + select { + case healthy := <-done: + if healthy { + t.Fatal("expected unhealthy after the grace period") + } + case <-time.After(5 * time.Second): + t.Fatal("expired-absence probe waited on the lock-ring mutex") + } +} + +func TestLockRingAbsenceRingUpdatesDuringProbe(t *testing.T) { + self := pb.ServerAddress("filer-0:8888") + other := pb.ServerAddress("filer-1:8888") + dlm := lock_manager.NewDistributedLockManager(self) + fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{Host: self}} + dlm.LockRing.SetTakeSnapshotCallback(fs.OnDlmChangeSnapshot) + var version int64 + install := func(servers ...pb.ServerAddress) { + version++ + dlm.LockRing.SetSnapshot(servers, version) + } + rejoinAndLeave := func() { install(self, other); install(other) } + probe := func(beforeRead, afterRead func()) time.Duration { + return fs.lockRingAbsence(func() bool { + if beforeRead != nil { + beforeRead() + } + in := slices.Contains(dlm.LockRing.Ring.GetServers(), self) + if afterRead != nil { + afterRead() + } + return in + }) + } + expired := func() { + install(other) + fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano()) + } + + expired() + if absent := probe(nil, func() { install(self, other) }); absent != 0 || fs.lockRingUnhealthySince.Load() > 0 { + t.Fatalf("rejoin after the read: absent %v, clock %d; want 0 and no clock", absent, fs.lockRingUnhealthySince.Load()) + } + + expired() + if absent := probe(rejoinAndLeave, nil); absent >= lockRingUnhealthyGracePeriod { + t.Fatalf("rejoin and removal before the read: absent %v, want a fresh grace period", absent) + } + + expired() + if absent := probe(nil, rejoinAndLeave); absent >= lockRingUnhealthyGracePeriod || fs.lockRingUnhealthySince.Load() > 0 { + t.Fatalf("rejoin and removal after the read: absent %v, clock %d; want a fresh grace period", absent, fs.lockRingUnhealthySince.Load()) + } + + install(other) + fs.resetLockRingClock() + var left int64 + probe(nil, func() { rejoinAndLeave(); left = time.Now().UnixNano() }) + if since := fs.lockRingUnhealthySince.Load(); since > 0 && since < left { + t.Fatalf("clock started at %d, before the removal at %d", since, left) + } +}