diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 156ecdf0a2..f178ba5506 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -487,7 +487,12 @@ func shouldInvalidateConnection(ctx context.Context, err error) bool { if s, ok := status.FromError(err); ok { code := s.Code() switch code { - case codes.Unavailable, codes.Aborted, codes.Internal: + case codes.Unavailable: + // gRPC raises its own Unavailable without details, so one that + // carries details is the peer answering over a working channel, + // e.g. a master follower turning down an assign. + return len(s.Proto().GetDetails()) == 0 + case codes.Aborted, codes.Internal: return true case codes.ResourceExhausted: // Server-side backpressure on a healthy channel (e.g. the master's diff --git a/weed/pb/grpc_client_server_test.go b/weed/pb/grpc_client_server_test.go index 8bcf4e6cf3..86e71e44c0 100644 --- a/weed/pb/grpc_client_server_test.go +++ b/weed/pb/grpc_client_server_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -127,6 +128,22 @@ func TestShouldInvalidateConnection_PeerAnswerNamingTransportKeepsChannel(t *tes } } +func TestShouldInvalidateConnection_UnavailableAnswerKeepsChannel(t *testing.T) { + st, err := status.New(codes.Unavailable, "raft.Server: Not current leader").WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER"}) + if err != nil { + t.Fatal(err) + } + if shouldInvalidateConnection(context.Background(), st.Err()) { + t.Errorf("Unavailable answered by the peer must not invalidate the shared connection") + } + if shouldInvalidateConnection(context.Background(), fmt.Errorf("assign: %w", st.Err())) { + t.Errorf("wrapped Unavailable answered by the peer must not invalidate the shared connection") + } + if !shouldInvalidateConnection(context.Background(), status.Error(codes.Unavailable, "connection error: desc = \"transport: Error while dialing\"")) { + t.Errorf("Unavailable raised by gRPC itself must still invalidate the shared connection") + } +} + // TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the // marshal-error carve-out does not swallow real server-side Internal errors, // which previously caused — and should continue to cause — connection diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 4339d7732b..37a9a41372 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -53,7 +53,7 @@ func (ms *MasterServer) StreamAssign(server master_pb.Seaweed_StreamAssignServer func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) { if !ms.Topo.IsLeader() { - return nil, raft.NotLeaderError + return nil, notLeaderError() } if req.Count == 0 { @@ -333,6 +333,15 @@ func (w *unservedLayoutWarning) Do(optionKey string, lastErr error) { glog.Warningf("assign requests for %s will keep failing until a volume server registers that disk layout or clients change their assignment disk type: %v", optionKey, lastErr) } +func notLeaderError() error { + st := status.New(codes.Unavailable, raft.NotLeaderError.Error()) + withReason, err := st.WithDetails(&errdetails.ErrorInfo{Reason: "NOT_LEADER", Domain: "seaweedfs"}) + if err != nil { + return st.Err() + } + return withReason.Err() +} + // warmingUpError carries the remaining warm-up as a RetryInfo detail, the gRPC // counterpart of the HTTP handlers' Retry-After. func warmingUpError(remaining time.Duration) error { diff --git a/weed/server/master_grpc_server_assign_test.go b/weed/server/master_grpc_server_assign_test.go index a712147659..9243c10251 100644 --- a/weed/server/master_grpc_server_assign_test.go +++ b/weed/server/master_grpc_server_assign_test.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -41,6 +42,31 @@ func newLeaderMaster() *MasterServer { } } +// followerRaftServer is a raft follower that knows no leader yet, as during an +// election. +type followerRaftServer struct { + raft.Server +} + +func (followerRaftServer) State() string { return raft.Follower } +func (followerRaftServer) Leader() string { return "" } +func (followerRaftServer) Name() string { return "127.0.0.1:9333" } + +func TestAssignOnFollowerIsUnavailable(t *testing.T) { + ms := newLeaderMaster() + ms.Topo.RaftServer = followerRaftServer{} + + _, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1}) + st, ok := status.FromError(err) + require.True(t, ok, "want a gRPC status, got %v", err) + assert.Equal(t, codes.Unavailable, st.Code()) + assert.Equal(t, raft.NotLeaderError.Error(), st.Message()) + require.Len(t, st.Details(), 1) + info, ok := st.Details()[0].(*errdetails.ErrorInfo) + require.True(t, ok, "want an ErrorInfo detail, got %T", st.Details()[0]) + assert.Equal(t, "NOT_LEADER", info.GetReason()) +} + // markGrowthInFlight flags growth on the same VolumeLayout the handler resolves, // by mirroring its exact lookup. func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {