From 6bdcd17ed44c807a5b555608c750e582562be823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=A9=AC=E4=B8=80=E5=8F=AA?= Date: Fri, 20 Nov 2026 05:11:40 +0800 Subject: [PATCH] master: reject an assign on a follower as Unavailable (#12779) * master: reject an assign on a follower as Unavailable Assign on a master that is not the raft leader returned raft.NotLeaderError as a plain error, which reaches gRPC clients as codes.Unknown. operation.Assign retries only Unavailable, ResourceExhausted, Canceled and DeadlineExceeded, so every assign made during a leader election failed at once instead of waiting for the new leader. Return it as codes.Unavailable, like the warming-up status in the same file, so clients retry within their 30 s assign budget. The message is unchanged, so callers that match "Not current leader" still work. StreamAssign now answers such a request in-band with the error message instead of ending the stream, as it already does for warm-up. TestAssignOnFollowerIsUnavailable covers it. Co-Authored-By: Claude Opus 5.5 * pb: keep the cached channel when the peer answers Unavailable with details gRPC's own Unavailable (dial failure, closing transport) never carries details, so one that does came from the peer over a working channel. Closing the shared ClientConn for it cancels every other in-flight RPC on it with "the client connection is closing". Co-Authored-By: Claude Opus 5.5 * master: tag the follower's assign rejection with a NOT_LEADER ErrorInfo The detail marks it as the master's answer, so clients retry it without tearing down their cached channel to that master. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- weed/pb/grpc_client_server.go | 7 ++++- weed/pb/grpc_client_server_test.go | 17 ++++++++++++ weed/server/master_grpc_server_assign.go | 11 +++++++- weed/server/master_grpc_server_assign_test.go | 26 +++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 156ecdf0a2..f178ba5506 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -487,7 +487,12 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool { if s, ok := status.FromError(err); ok { code := s.Code() switch code { - case codes.Unavailable, codes.Aborted, codes.Internal: + case codes.Unavailable: + // gRPC raises its own Unavailable without details, so one that + // carries details is the peer answering over a working channel, + // e.g. a master follower turning down an assign. + return len(s.Proto().GetDetails()) == 0 + case codes.Aborted, codes.Internal: return true case codes.ResourceExhausted: // Server-side backpressure on a healthy channel (e.g. the master's diff --git a/weed/pb/grpc_client_server_test.go b/weed/pb/grpc_client_server_test.go index 8bcf4e6cf3..86e71e44c0 100644 --- a/weed/pb/grpc_client_server_test.go +++ b/weed/pb/grpc_client_server_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -127,6 +128,22 @@ func TestShouldInvalidateConnection_PeerAnswerNamingTransportKeepsChannel(t *tes } } +func TestShouldInvalidateConnection_UnavailableAnswerKeepsChannel(t *testing.T) { + st, err := status.New(codes.Unavailable, "raft.Server: Not current leader").WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER"}) + if err != nil { + t.Fatal(err) + } + if shouldInvalidateConnection(context.Background(), st.Err()) { + t.Errorf("Unavailable answered by the peer must not invalidate the shared connection") + } + if shouldInvalidateConnection(context.Background(), fmt.Errorf("assign: %w", st.Err())) { + t.Errorf("wrapped Unavailable answered by the peer must not invalidate the shared connection") + } + if !shouldInvalidateConnection(context.Background(), status.Error(codes.Unavailable, "connection error: desc = \"transport: Error while dialing\"")) { + t.Errorf("Unavailable raised by gRPC itself must still invalidate the shared connection") + } +} + // TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the // marshal-error carve-out does not swallow real server-side Internal errors, // which previously caused — and should continue to cause — connection diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 4339d7732b..37a9a41372 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -53,7 +53,7 @@ func (ms *MasterServer) StreamAssign(server master_pb.Seaweed_StreamAssignServer func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) { if !ms.Topo.IsLeader() { - return nil, raft.NotLeaderError + return nil, notLeaderError() } if req.Count == 0 { @@ -333,6 +333,15 @@ func (w *unservedLayoutWarning) Do(optionKey string, lastErr error) { glog.Warningf("assign requests for %s will keep failing until a volume server registers that disk layout or clients change their assignment disk type: %v", optionKey, lastErr) } +func notLeaderError() error { + st := status.New(codes.Unavailable, raft.NotLeaderError.Error()) + withReason, err := st.WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER", Domain: "seaweedfs"}) + if err != nil { + return st.Err() + } + return withReason.Err() +} + // warmingUpError carries the remaining warm-up as a RetryInfo detail, the gRPC // counterpart of the HTTP handlers' Retry-After. func warmingUpError(remaining time.Duration) error { diff --git a/weed/server/master_grpc_server_assign_test.go b/weed/server/master_grpc_server_assign_test.go index a712147659..9243c10251 100644 --- a/weed/server/master_grpc_server_assign_test.go +++ b/weed/server/master_grpc_server_assign_test.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -41,6 +42,31 @@ func newLeaderMaster() *MasterServer { } } +// followerRaftServer is a raft follower that knows no leader yet, as during an +// election. +type followerRaftServer struct { + raft.Server +} + +func (followerRaftServer) State() string { return raft.Follower } +func (followerRaftServer) Leader() string { return "" } +func (followerRaftServer) Name() string { return "127.0.0.1:9333" } + +func TestAssignOnFollowerIsUnavailable(t *testing.T) { + ms := newLeaderMaster() + ms.Topo.RaftServer = followerRaftServer{} + + _, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1}) + st, ok := status.FromError(err) + require.True(t, ok, "want a gRPC status, got %v", err) + assert.Equal(t, codes.Unavailable, st.Code()) + assert.Equal(t, raft.NotLeaderError.Error(), st.Message()) + require.Len(t, st.Details(), 1) + info, ok := st.Details()[0].(*errdetails.ErrorInfo) + require.True(t, ok, "want an ErrorInfo detail, got %T", st.Details()[0]) + assert.Equal(t, "NOT_LEADER", info.GetReason()) +} + // markGrowthInFlight flags growth on the same VolumeLayout the handler resolves, // by mirroring its exact lookup. func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {