mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 17:55:26 +01:00
filer: fail health check when absent from lock ring (#12709)
* filer: fail health check when absent from lock ring If the master broadcasts a lock ring that does not include this filer (e.g. the late-cleanup race in seaweedfs#11442), every lock acquisition fails with "no lock server found" and locked writes, including S3 PUTs, hang indefinitely while /healthz and /readyz still report OK. Make the filer health handler return 503 once this filer has been missing from its own lock ring for longer than a 30s grace period, so Kubernetes liveness probes restart it and it re-registers with the master. The grace period covers startup before the first ring update and the master's ring-change batching delay. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: hold the lock-ring health clock until the filer announces AnnounceOnceCaughtUp can hold the master registration for up to a minute while HTTP already serves, so the filer cannot be in the ring yet. Report healthy until the announce gate opens and start the grace period after it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep a probe racing a ring rejoin from reading a reset clock A probe that loses the CompareAndSwap to a concurrent healthy probe could load the reset zero and report the filer absent since the epoch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: trim the lock-ring health comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: restart the lock-ring health grace when a ring update brings the filer back The clock only reset when a probe saw the filer in the ring, so a rejoin and a second removal between two probes reused the first absence and failed the next probe early. Reset it from the ring snapshot callback. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test the announce gate leaves the lock-ring health clock unset Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: re-read lock-ring membership after starting the health clock A probe whose first read raced a rejoin could start the clock after the snapshot callback reset it, leaving a stale start for the next removal. The ring is installed before the callback runs, so a second read sees the rejoin whenever the reset came first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: treat a health clock changed during the probe as a rejoin A rejoin and a second removal can both land between the probe's two membership reads. The second read then shows the filer absent again, and the probe judged the new absence by the expired start it had read before the rejoin's reset. The snapshot callback resets the clock before the next ring update is installed, so a clock that no longer matches the probe's start means a rejoin happened; report healthy and let the next probe use the fresh clock. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test ring updates that land during a lock-ring health probe Move the absence clock into lockRingAbsence, which takes the membership read as a function, so the test can install ring updates between the probe's reads: a rejoin after the first read, and a rejoin plus removal between the two reads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: give every lock-ring health reset a new clock value A probe that saw the filer absent started the clock with a CAS from 0, so a rejoin that reset the clock to 0 again before that CAS was invisible, and the probe could install a start time from before the rejoin. Each reset now stores a new non-positive value and the probe swaps from the value it loaded before the membership read, so any rejoin during the probe makes the CAS fail. This drops the second membership read and never waits on the LockRing mutex. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test lock-ring rejoins before and after the probe's membership read Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep the lock-ring version lookup out of the unhealthy health path LockRing.Version takes the ring mutex, which SetSnapshot holds while the snapshot callback transfers locks, so an expired-absence probe could wait on those transfers and time out instead of answering 503. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test an expired-absence health probe while the ring mutex is held Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
4 files changed
+222
-2
No files matched your search
@@ -3,6 +3,7 @@ package weed_server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -178,6 +179,9 @@ func (fs *FilerServer) ReplicateLock(ctx context.Context, req *filer_pb.Replicat
|
||||
// 2. Re-replicating to the new backup
|
||||
// 3. Transferring locks that no longer belong here
|
||||
func (fs *FilerServer) OnDlmChangeSnapshot(snapshot []pb.ServerAddress) {
|
||||
if slices.Contains(snapshot, fs.option.Host) {
|
||||
fs.resetLockRingClock()
|
||||
}
|
||||
locks := fs.filer.Dlm.AllLocks()
|
||||
if len(locks) == 0 {
|
||||
return
|
||||
|
||||
@@ -120,6 +120,10 @@ type FilerOption struct {
|
||||
type FilerServer struct {
|
||||
inFlightDataSize int64
|
||||
inFlightUploads int64
|
||||
// unix nanos when this filer was first seen missing from its own lock ring;
|
||||
// <= 0 while it is in the ring, with a new value on every reset
|
||||
lockRingUnhealthySince atomic.Int64
|
||||
lockRingResets atomic.Int64
|
||||
|
||||
inFlightDataLimitCond *sync.Cond
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -390,12 +391,68 @@ func pathHasComponentPrefix(reqPath, prefix string) bool {
|
||||
return strings.HasPrefix(cleanedPath, cleanedPrefix+"/")
|
||||
}
|
||||
|
||||
// lockRingUnhealthyGracePeriod covers the first ring update after announcing
|
||||
// and the master's ring-change batching delay.
|
||||
const lockRingUnhealthyGracePeriod = 30 * time.Second
|
||||
|
||||
// lockRingHealthy reports false once this filer has been missing from its own
|
||||
// lock ring past the grace period: locked writes cannot be routed to it, and
|
||||
// a restart re-registers it with the master.
|
||||
func (fs *FilerServer) lockRingHealthy() bool {
|
||||
dlm := fs.filer.Dlm
|
||||
if dlm == nil || dlm.LockRing == nil || dlm.LockRing.Ring == nil {
|
||||
return true
|
||||
}
|
||||
if announce := fs.option.AnnounceCh; announce != nil {
|
||||
select {
|
||||
case <-announce:
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
absent := fs.lockRingAbsence(func() bool { return slices.Contains(dlm.LockRing.Ring.GetServers(), dlm.Host) })
|
||||
if absent < lockRingUnhealthyGracePeriod {
|
||||
return true
|
||||
}
|
||||
glog.Warningf("filerHealthz: %s absent from lock ring for %v (ring: %v)",
|
||||
dlm.Host, absent.Round(time.Second), dlm.LockRing.Ring.GetServers())
|
||||
return false
|
||||
}
|
||||
|
||||
// lockRingAbsence returns how long inRing has reported this filer missing,
|
||||
// or 0 while it is in the ring.
|
||||
func (fs *FilerServer) lockRingAbsence(inRing func() bool) time.Duration {
|
||||
since := fs.lockRingUnhealthySince.Load()
|
||||
if inRing() {
|
||||
fs.resetLockRingClock()
|
||||
return 0
|
||||
}
|
||||
// A rejoin after the load has stored a new reset value, so the clock
|
||||
// neither starts nor is read across it.
|
||||
if since <= 0 {
|
||||
fs.lockRingUnhealthySince.CompareAndSwap(since, time.Now().UnixNano())
|
||||
return 0
|
||||
}
|
||||
if fs.lockRingUnhealthySince.Load() != since {
|
||||
return 0
|
||||
}
|
||||
return time.Since(time.Unix(0, since))
|
||||
}
|
||||
|
||||
func (fs *FilerServer) resetLockRingClock() {
|
||||
fs.lockRingUnhealthySince.Store(-fs.lockRingResets.Add(1))
|
||||
}
|
||||
|
||||
func (fs *FilerServer) filerHealthzHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Server", "SeaweedFS "+version.VERSION)
|
||||
if _, err := fs.filer.Store.FindEntry(context.Background(), filer.TopicsDir); err != nil && err != filer_pb.ErrNotFound {
|
||||
glog.Warningf("filerHealthzHandler FindEntry: %+v", err)
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if !fs.lockRingHealthy() {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
)
|
||||
|
||||
func TestLockRingHealthy(t *testing.T) {
|
||||
self := pb.ServerAddress("filer-0:8888")
|
||||
dlm := lock_manager.NewDistributedLockManager(self)
|
||||
fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{}}
|
||||
|
||||
// Empty ring, within grace period: healthy.
|
||||
if !fs.lockRingHealthy() {
|
||||
t.Fatal("expected healthy during grace period")
|
||||
}
|
||||
|
||||
// Empty ring past the grace period: unhealthy.
|
||||
fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano())
|
||||
if fs.lockRingHealthy() {
|
||||
t.Fatal("expected unhealthy after grace period with empty ring")
|
||||
}
|
||||
|
||||
// Ring without self past the grace period: still unhealthy.
|
||||
dlm.LockRing.SetSnapshot([]pb.ServerAddress{"filer-1:8888"}, 1)
|
||||
if fs.lockRingHealthy() {
|
||||
t.Fatal("expected unhealthy when self is not in ring")
|
||||
}
|
||||
|
||||
// Self back in ring: healthy and timer reset.
|
||||
dlm.LockRing.SetSnapshot([]pb.ServerAddress{self}, 2)
|
||||
if !fs.lockRingHealthy() {
|
||||
t.Fatal("expected healthy when self is in ring")
|
||||
}
|
||||
if fs.lockRingUnhealthySince.Load() > 0 {
|
||||
t.Fatal("expected unhealthy timer to reset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockRingHealthyWaitsForAnnounce(t *testing.T) {
|
||||
dlm := lock_manager.NewDistributedLockManager("filer-0:8888")
|
||||
announce := make(chan struct{})
|
||||
fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{AnnounceCh: announce}}
|
||||
expired := time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano()
|
||||
|
||||
if !fs.lockRingHealthy() || fs.lockRingUnhealthySince.Load() > 0 {
|
||||
t.Fatal("expected healthy with no absence clock while registration is held")
|
||||
}
|
||||
fs.lockRingUnhealthySince.Store(expired)
|
||||
if !fs.lockRingHealthy() {
|
||||
t.Fatal("expected healthy while registration is held")
|
||||
}
|
||||
|
||||
close(announce)
|
||||
fs.lockRingUnhealthySince.Store(0)
|
||||
if !fs.lockRingHealthy() || fs.lockRingUnhealthySince.Load() <= 0 {
|
||||
t.Fatal("expected the first probe after announcing to start the grace period")
|
||||
}
|
||||
fs.lockRingUnhealthySince.Store(expired)
|
||||
if fs.lockRingHealthy() {
|
||||
t.Fatal("expected unhealthy after announcing with an empty ring")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockRingHealthyRestartsGraceAfterUnprobedRejoin(t *testing.T) {
|
||||
self := pb.ServerAddress("filer-0:8888")
|
||||
dlm := lock_manager.NewDistributedLockManager(self)
|
||||
fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{Host: self}}
|
||||
dlm.LockRing.SetTakeSnapshotCallback(fs.OnDlmChangeSnapshot)
|
||||
|
||||
fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano())
|
||||
dlm.LockRing.SetSnapshot([]pb.ServerAddress{self, "filer-1:8888"}, 1)
|
||||
dlm.LockRing.SetSnapshot([]pb.ServerAddress{"filer-1:8888"}, 2)
|
||||
if !fs.lockRingHealthy() {
|
||||
t.Fatal("expected a fresh grace period after rejoining the ring between probes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockRingHealthyDoesNotWaitOnRingMutex(t *testing.T) {
|
||||
dlm := lock_manager.NewDistributedLockManager("filer-0:8888")
|
||||
fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{}}
|
||||
dlm.LockRing.SetSnapshot([]pb.ServerAddress{"filer-1:8888"}, 1)
|
||||
fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano())
|
||||
|
||||
dlm.LockRing.Lock()
|
||||
defer dlm.LockRing.Unlock()
|
||||
done := make(chan bool, 1)
|
||||
go func() { done <- fs.lockRingHealthy() }()
|
||||
select {
|
||||
case healthy := <-done:
|
||||
if healthy {
|
||||
t.Fatal("expected unhealthy after the grace period")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("expired-absence probe waited on the lock-ring mutex")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockRingAbsenceRingUpdatesDuringProbe(t *testing.T) {
|
||||
self := pb.ServerAddress("filer-0:8888")
|
||||
other := pb.ServerAddress("filer-1:8888")
|
||||
dlm := lock_manager.NewDistributedLockManager(self)
|
||||
fs := &FilerServer{filer: &filer.Filer{Dlm: dlm}, option: &FilerOption{Host: self}}
|
||||
dlm.LockRing.SetTakeSnapshotCallback(fs.OnDlmChangeSnapshot)
|
||||
var version int64
|
||||
install := func(servers ...pb.ServerAddress) {
|
||||
version++
|
||||
dlm.LockRing.SetSnapshot(servers, version)
|
||||
}
|
||||
rejoinAndLeave := func() { install(self, other); install(other) }
|
||||
probe := func(beforeRead, afterRead func()) time.Duration {
|
||||
return fs.lockRingAbsence(func() bool {
|
||||
if beforeRead != nil {
|
||||
beforeRead()
|
||||
}
|
||||
in := slices.Contains(dlm.LockRing.Ring.GetServers(), self)
|
||||
if afterRead != nil {
|
||||
afterRead()
|
||||
}
|
||||
return in
|
||||
})
|
||||
}
|
||||
expired := func() {
|
||||
install(other)
|
||||
fs.lockRingUnhealthySince.Store(time.Now().Add(-lockRingUnhealthyGracePeriod - time.Second).UnixNano())
|
||||
}
|
||||
|
||||
expired()
|
||||
if absent := probe(nil, func() { install(self, other) }); absent != 0 || fs.lockRingUnhealthySince.Load() > 0 {
|
||||
t.Fatalf("rejoin after the read: absent %v, clock %d; want 0 and no clock", absent, fs.lockRingUnhealthySince.Load())
|
||||
}
|
||||
|
||||
expired()
|
||||
if absent := probe(rejoinAndLeave, nil); absent >= lockRingUnhealthyGracePeriod {
|
||||
t.Fatalf("rejoin and removal before the read: absent %v, want a fresh grace period", absent)
|
||||
}
|
||||
|
||||
expired()
|
||||
if absent := probe(nil, rejoinAndLeave); absent >= lockRingUnhealthyGracePeriod || fs.lockRingUnhealthySince.Load() > 0 {
|
||||
t.Fatalf("rejoin and removal after the read: absent %v, clock %d; want a fresh grace period", absent, fs.lockRingUnhealthySince.Load())
|
||||
}
|
||||
|
||||
install(other)
|
||||
fs.resetLockRingClock()
|
||||
var left int64
|
||||
probe(nil, func() { rejoinAndLeave(); left = time.Now().UnixNano() })
|
||||
if since := fs.lockRingUnhealthySince.Load(); since > 0 && since < left {
|
||||
t.Fatalf("clock started at %d, before the removal at %d", since, left)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user