fix(filer): leave the lock ring before stopping gRPC on shutdown (#11604)

A draining filer stayed in the master's lock ring until its process
exited, while gRPC GracefulStop was already refusing new connections. S3
gateways and peer filers kept routing object-write locks and owner-routed
writes to it for the whole graceful-stop window.

On shutdown the filer now sends leave_lock_ring on its open KeepConnected
stream. The master removes it from the lock ring only; it stays a cluster
member so peers keep following its metadata log through the drain. Once
the ring update without it arrives, the filer has already transferred its
locks to the new owners, and it keeps serving through the prior-owner
window (plus a second for peers that apply the update later) before gRPC
and HTTP begin draining. The whole leave is bounded at 10s so slow lock
transfers or a stuck stream cannot hold up the drain. A lone filer, or a
master that ignores the message, falls back to the previous behavior.

weed/command/filer.go

@@ -588,7 +588,7 @@ func (fo *FilerOptions) startFiler() {
 		}
 		httpS := newHttpServer(defaultHandler, tlsConfig)
 		httpServers = append(httpServers, httpS)
-		shutdown := newFilerShutdown(stopGrpcServer, fs.Shutdown, httpServers...)
+		shutdown := newFilerShutdown(fs.LeaveLockRing, stopGrpcServer, fs.Shutdown, httpServers...)
 
 		grace.OnInterrupt(shutdown)
 
@@ -617,7 +617,7 @@ func (fo *FilerOptions) startFiler() {
 		}
 		httpS := newHttpServer(defaultHandler, nil)
 		httpServers = append(httpServers, httpS)
-		shutdown := newFilerShutdown(stopGrpcServer, fs.Shutdown, httpServers...)
+		shutdown := newFilerShutdown(fs.LeaveLockRing, stopGrpcServer, fs.Shutdown, httpServers...)
 
 		grace.OnInterrupt(shutdown)
 
@@ -637,8 +637,11 @@ func (fo *FilerOptions) startFiler() {
 }
 
 // newFilerShutdown joins shutdown callers while gRPC and HTTP drain concurrently.
-func newFilerShutdown(stopGrpc, shutdownFiler func(), httpServers ...*http.Server) func() {
+// The filer leaves the lock ring first: peers and S3 gateways route keys to it
+// until the ring changes, and would hit refused connections once gRPC stops.
+func newFilerShutdown(leaveLockRing, stopGrpc, shutdownFiler func(), httpServers ...*http.Server) func() {
 	return sync.OnceFunc(func() {
+		leaveLockRing()
 		shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
 		defer cancel()
 		var drained sync.WaitGroup

weed/command/filer_shutdown_test.go

@@ -56,7 +56,7 @@ func TestFilerShutdownJoinsServeDuringParallelDrain(t *testing.T) {
 	grpcStopped := make(chan struct{})
 	filerClosed := make(chan struct{})
 	var closes atomic.Int32
-	shutdown := newFilerShutdown(func() {
+	shutdown := newFilerShutdown(func() {}, func() {
 		close(grpcStarted)
 		<-grpcRelease
 		close(grpcStopped)
@@ -138,7 +138,7 @@ func TestFilerShutdownWaitsForHTTPAfterGrpcStops(t *testing.T) {
 	server.Config.RegisterOnShutdown(func() { close(httpClosing) })
 	grpcStopped := make(chan struct{})
 	filerClosed := make(chan struct{})
-	shutdown := newFilerShutdown(func() { close(grpcStopped) }, func() { close(filerClosed) }, server.Config)
+	shutdown := newFilerShutdown(func() {}, func() { close(grpcStopped) }, func() { close(filerClosed) }, server.Config)
 	joined := make(chan struct{})
 	go func() { shutdown(); close(joined) }()
 	<-httpClosing
@@ -157,3 +157,42 @@ func TestFilerShutdownWaitsForHTTPAfterGrpcStops(t *testing.T) {
 		t.Error("filer was not closed after HTTP request completed")
 	}
 }
+
+func TestFilerShutdownLeavesLockRingBeforeDraining(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {}))
+	t.Cleanup(server.Close)
+	httpClosing := make(chan struct{})
+	server.Config.RegisterOnShutdown(func() { close(httpClosing) })
+
+	leaving := make(chan struct{})
+	releaseLeave := make(chan struct{})
+	finishLeave := sync.OnceFunc(func() { close(releaseLeave) })
+	t.Cleanup(finishLeave)
+	grpcStopping := make(chan struct{})
+	shutdown := newFilerShutdown(func() {
+		close(leaving)
+		<-releaseLeave
+	}, func() { close(grpcStopping) }, func() {}, server.Config)
+	joined := make(chan struct{})
+	go func() { shutdown(); close(joined) }()
+
+	select {
+	case <-leaving:
+	case <-time.After(5 * time.Second):
+		t.Fatal("shutdown did not leave the lock ring")
+	}
+	select {
+	case <-grpcStopping:
+		t.Fatal("gRPC stopped accepting while the filer was still leaving the lock ring")
+	case <-httpClosing:
+		t.Fatal("HTTP stopped accepting while the filer was still leaving the lock ring")
+	case <-time.After(100 * time.Millisecond):
+	}
+	finishLeave()
+	<-joined
+	select {
+	case <-grpcStopping:
+	default:
+		t.Error("gRPC was not stopped after leaving the lock ring")
+	}
+}