master: reject an assign on a follower as Unavailable (#12779)

* master: reject an assign on a follower as Unavailable

Assign on a master that is not the raft leader returned
raft.NotLeaderError as a plain error, which reaches gRPC clients as
codes.Unknown. operation.Assign retries only Unavailable,
ResourceExhausted, Canceled and DeadlineExceeded, so every assign made
during a leader election failed at once instead of waiting for the new
leader.

Return it as codes.Unavailable, like the warming-up status in the same
file, so clients retry within their 30 s assign budget. The message is
unchanged, so callers that match "Not current leader" still work.
StreamAssign now answers such a request in-band with the error message
instead of ending the stream, as it already does for warm-up.

TestAssignOnFollowerIsUnavailable covers it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* pb: keep the cached channel when the peer answers Unavailable with details

gRPC's own Unavailable (dial failure, closing transport) never carries
details, so one that does came from the peer over a working channel.
Closing the shared ClientConn for it cancels every other in-flight RPC
on it with "the client connection is closing".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: tag the follower's assign rejection with a NOT_LEADER ErrorInfo

The detail marks it as the master's answer, so clients retry it
without tearing down their cached channel to that master.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 13:11:40 -08:00
1 parent 729b872f01
commit 6bdcd17ed4
4 files changed
+59 -2

No files matched your search

+6 -1
View File
@@ -487,7 +487,12 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool {
if s, ok := status.FromError(err); ok {
code := s.Code()
switch code {
case codes.Unavailable, codes.Aborted, codes.Internal:
case codes.Unavailable:
// gRPC raises its own Unavailable without details, so one that
// carries details is the peer answering over a working channel,
// e.g. a master follower turning down an assign.
return len(s.Proto().GetDetails()) == 0
case codes.Aborted, codes.Internal:
return true
case codes.ResourceExhausted:
// Server-side backpressure on a healthy channel (e.g. the master's
+17
View File
@@ -6,6 +6,7 @@ import (
"runtime"
"testing"
"google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -127,6 +128,22 @@ func TestShouldInvalidateConnection_PeerAnswerNamingTransportKeepsChannel(t *tes
}
}
func TestShouldInvalidateConnection_UnavailableAnswerKeepsChannel(t *testing.T) {
st, err := status.New(codes.Unavailable, "raft.Server: Not current leader").WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER"})
if err != nil {
t.Fatal(err)
}
if shouldInvalidateConnection(context.Background(), st.Err()) {
t.Errorf("Unavailable answered by the peer must not invalidate the shared connection")
}
if shouldInvalidateConnection(context.Background(), fmt.Errorf("assign: %w", st.Err())) {
t.Errorf("wrapped Unavailable answered by the peer must not invalidate the shared connection")
}
if !shouldInvalidateConnection(context.Background(), status.Error(codes.Unavailable, "connection error: desc = \"transport: Error while dialing\"")) {
t.Errorf("Unavailable raised by gRPC itself must still invalidate the shared connection")
}
}
// TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the
// marshal-error carve-out does not swallow real server-side Internal errors,
// which previously caused — and should continue to cause — connection
+10 -1
View File
@@ -53,7 +53,7 @@ func (ms *MasterServer) StreamAssign(server master_pb.Seaweed_StreamAssignServer
func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) {
if !ms.Topo.IsLeader() {
return nil, raft.NotLeaderError
return nil, notLeaderError()
}
if req.Count == 0 {
@@ -333,6 +333,15 @@ func (w *unservedLayoutWarning) Do(optionKey string, lastErr error) {
glog.Warningf("assign requests for %s will keep failing until a volume server registers that disk layout or clients change their assignment disk type: %v", optionKey, lastErr)
}
func notLeaderError() error {
st := status.New(codes.Unavailable, raft.NotLeaderError.Error())
withReason, err := st.WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER", Domain: "seaweedfs"})
if err != nil {
return st.Err()
}
return withReason.Err()
}
// warmingUpError carries the remaining warm-up as a RetryInfo detail, the gRPC
// counterpart of the HTTP handlers' Retry-After.
func warmingUpError(remaining time.Duration) error {
@@ -9,6 +9,7 @@ import (
"github.com/seaweedfs/raft"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -41,6 +42,31 @@ func newLeaderMaster() *MasterServer {
}
}
// followerRaftServer is a raft follower that knows no leader yet, as during an
// election.
type followerRaftServer struct {
raft.Server
}
func (followerRaftServer) State() string { return raft.Follower }
func (followerRaftServer) Leader() string { return "" }
func (followerRaftServer) Name() string { return "127.0.0.1:9333" }
func TestAssignOnFollowerIsUnavailable(t *testing.T) {
ms := newLeaderMaster()
ms.Topo.RaftServer = followerRaftServer{}
_, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1})
st, ok := status.FromError(err)
require.True(t, ok, "want a gRPC status, got %v", err)
assert.Equal(t, codes.Unavailable, st.Code())
assert.Equal(t, raft.NotLeaderError.Error(), st.Message())
require.Len(t, st.Details(), 1)
info, ok := st.Details()[0].(*errdetails.ErrorInfo)
require.True(t, ok, "want an ErrorInfo detail, got %T", st.Details()[0])
assert.Equal(t, "NOT_LEADER", info.GetReason())
}
// markGrowthInFlight flags growth on the same VolumeLayout the handler resolves,
// by mirroring its exact lookup.
func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {