mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
master: reject an assign on a follower as Unavailable (#12779)
* master: reject an assign on a follower as Unavailable Assign on a master that is not the raft leader returned raft.NotLeaderError as a plain error, which reaches gRPC clients as codes.Unknown. operation.Assign retries only Unavailable, ResourceExhausted, Canceled and DeadlineExceeded, so every assign made during a leader election failed at once instead of waiting for the new leader. Return it as codes.Unavailable, like the warming-up status in the same file, so clients retry within their 30 s assign budget. The message is unchanged, so callers that match "Not current leader" still work. StreamAssign now answers such a request in-band with the error message instead of ending the stream, as it already does for warm-up. TestAssignOnFollowerIsUnavailable covers it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * pb: keep the cached channel when the peer answers Unavailable with details gRPC's own Unavailable (dial failure, closing transport) never carries details, so one that does came from the peer over a working channel. Closing the shared ClientConn for it cancels every other in-flight RPC on it with "the client connection is closing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: tag the follower's assign rejection with a NOT_LEADER ErrorInfo The detail marks it as the master's answer, so clients retry it without tearing down their cached channel to that master. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
4 files changed
+59
-2
No files matched your search
@@ -487,7 +487,12 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool {
|
||||
if s, ok := status.FromError(err); ok {
|
||||
code := s.Code()
|
||||
switch code {
|
||||
case codes.Unavailable, codes.Aborted, codes.Internal:
|
||||
case codes.Unavailable:
|
||||
// gRPC raises its own Unavailable without details, so one that
|
||||
// carries details is the peer answering over a working channel,
|
||||
// e.g. a master follower turning down an assign.
|
||||
return len(s.Proto().GetDetails()) == 0
|
||||
case codes.Aborted, codes.Internal:
|
||||
return true
|
||||
case codes.ResourceExhausted:
|
||||
// Server-side backpressure on a healthy channel (e.g. the master's
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
@@ -127,6 +128,22 @@ func TestShouldInvalidateConnection_PeerAnswerNamingTransportKeepsChannel(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldInvalidateConnection_UnavailableAnswerKeepsChannel(t *testing.T) {
|
||||
st, err := status.New(codes.Unavailable, "raft.Server: Not current leader").WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if shouldInvalidateConnection(context.Background(), st.Err()) {
|
||||
t.Errorf("Unavailable answered by the peer must not invalidate the shared connection")
|
||||
}
|
||||
if shouldInvalidateConnection(context.Background(), fmt.Errorf("assign: %w", st.Err())) {
|
||||
t.Errorf("wrapped Unavailable answered by the peer must not invalidate the shared connection")
|
||||
}
|
||||
if !shouldInvalidateConnection(context.Background(), status.Error(codes.Unavailable, "connection error: desc = \"transport: Error while dialing\"")) {
|
||||
t.Errorf("Unavailable raised by gRPC itself must still invalidate the shared connection")
|
||||
}
|
||||
}
|
||||
|
||||
// TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the
|
||||
// marshal-error carve-out does not swallow real server-side Internal errors,
|
||||
// which previously caused — and should continue to cause — connection
|
||||
|
||||
@@ -53,7 +53,7 @@ func (ms *MasterServer) StreamAssign(server master_pb.Seaweed_StreamAssignServer
|
||||
func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) {
|
||||
|
||||
if !ms.Topo.IsLeader() {
|
||||
return nil, raft.NotLeaderError
|
||||
return nil, notLeaderError()
|
||||
}
|
||||
|
||||
if req.Count == 0 {
|
||||
@@ -333,6 +333,15 @@ func (w *unservedLayoutWarning) Do(optionKey string, lastErr error) {
|
||||
glog.Warningf("assign requests for %s will keep failing until a volume server registers that disk layout or clients change their assignment disk type: %v", optionKey, lastErr)
|
||||
}
|
||||
|
||||
func notLeaderError() error {
|
||||
st := status.New(codes.Unavailable, raft.NotLeaderError.Error())
|
||||
withReason, err := st.WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER", Domain: "seaweedfs"})
|
||||
if err != nil {
|
||||
return st.Err()
|
||||
}
|
||||
return withReason.Err()
|
||||
}
|
||||
|
||||
// warmingUpError carries the remaining warm-up as a RetryInfo detail, the gRPC
|
||||
// counterpart of the HTTP handlers' Retry-After.
|
||||
func warmingUpError(remaining time.Duration) error {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/seaweedfs/raft"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
@@ -41,6 +42,31 @@ func newLeaderMaster() *MasterServer {
|
||||
}
|
||||
}
|
||||
|
||||
// followerRaftServer is a raft follower that knows no leader yet, as during an
|
||||
// election.
|
||||
type followerRaftServer struct {
|
||||
raft.Server
|
||||
}
|
||||
|
||||
func (followerRaftServer) State() string { return raft.Follower }
|
||||
func (followerRaftServer) Leader() string { return "" }
|
||||
func (followerRaftServer) Name() string { return "127.0.0.1:9333" }
|
||||
|
||||
func TestAssignOnFollowerIsUnavailable(t *testing.T) {
|
||||
ms := newLeaderMaster()
|
||||
ms.Topo.RaftServer = followerRaftServer{}
|
||||
|
||||
_, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1})
|
||||
st, ok := status.FromError(err)
|
||||
require.True(t, ok, "want a gRPC status, got %v", err)
|
||||
assert.Equal(t, codes.Unavailable, st.Code())
|
||||
assert.Equal(t, raft.NotLeaderError.Error(), st.Message())
|
||||
require.Len(t, st.Details(), 1)
|
||||
info, ok := st.Details()[0].(*errdetails.ErrorInfo)
|
||||
require.True(t, ok, "want an ErrorInfo detail, got %T", st.Details()[0])
|
||||
assert.Equal(t, "NOT_LEADER", info.GetReason())
|
||||
}
|
||||
|
||||
// markGrowthInFlight flags growth on the same VolumeLayout the handler resolves,
|
||||
// by mirroring its exact lookup.
|
||||
func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {
|
||||
|
||||
Reference in new issue
Block a user