Files
seaweedfs/weed/server/master_grpc_server_assign_test.go
T
6bdcd17ed4 master: reject an assign on a follower as Unavailable (#12779)
* master: reject an assign on a follower as Unavailable

Assign on a master that is not the raft leader returned
raft.NotLeaderError as a plain error, which reaches gRPC clients as
codes.Unknown. operation.Assign retries only Unavailable,
ResourceExhausted, Canceled and DeadlineExceeded, so every assign made
during a leader election failed at once instead of waiting for the new
leader.

Return it as codes.Unavailable, like the warming-up status in the same
file, so clients retry within their 30 s assign budget. The message is
unchanged, so callers that match "Not current leader" still work.
StreamAssign now answers such a request in-band with the error message
instead of ending the stream, as it already does for warm-up.

TestAssignOnFollowerIsUnavailable covers it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* pb: keep the cached channel when the peer answers Unavailable with details

gRPC's own Unavailable (dial failure, closing transport) never carries
details, so one that does came from the peer over a working channel.
Closing the shared ClientConn for it cancels every other in-flight RPC
on it with "the client connection is closing".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* master: tag the follower's assign rejection with a NOT_LEADER ErrorInfo

The detail marks it as the master's answer, so clients retry it
without tearing down their cached channel to that master.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2026-11-19 13:11:40 -08:00

507 lines
19 KiB
Go

package weed_server
import (
"context"
"fmt"
"testing"
"time"
"github.com/seaweedfs/raft"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/topology"
)
// leaderRaftServer answers only State(); the embedded nil interface panics if any
// other method is called.
type leaderRaftServer struct {
raft.Server
}
func (leaderRaftServer) State() string { return raft.Leader }
func newLeaderMaster() *MasterServer {
topo := topology.NewTopology("test", sequence.NewMemorySequencer(), 1<<20, 5, false)
topo.RaftServer = leaderRaftServer{}
return &MasterServer{
Topo: topo,
option: &MasterOption{},
guard: security.NewGuard(nil, "", 0, "", 0),
volumeGrowthRequestChan: make(chan *topology.VolumeGrowRequest, 1<<6),
}
}
// followerRaftServer is a raft follower that knows no leader yet, as during an
// election.
type followerRaftServer struct {
raft.Server
}
func (followerRaftServer) State() string { return raft.Follower }
func (followerRaftServer) Leader() string { return "" }
func (followerRaftServer) Name() string { return "127.0.0.1:9333" }
func TestAssignOnFollowerIsUnavailable(t *testing.T) {
ms := newLeaderMaster()
ms.Topo.RaftServer = followerRaftServer{}
_, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1})
st, ok := status.FromError(err)
require.True(t, ok, "want a gRPC status, got %v", err)
assert.Equal(t, codes.Unavailable, st.Code())
assert.Equal(t, raft.NotLeaderError.Error(), st.Message())
require.Len(t, st.Details(), 1)
info, ok := st.Details()[0].(*errdetails.ErrorInfo)
require.True(t, ok, "want an ErrorInfo detail, got %T", st.Details()[0])
assert.Equal(t, "NOT_LEADER", info.GetReason())
}
// markGrowthInFlight flags growth on the same VolumeLayout the handler resolves,
// by mirroring its exact lookup.
func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {
t.Helper()
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
require.NoError(t, err)
ttl, err := needle.ReadTTL(req.Ttl)
require.NoError(t, err)
topo.GetVolumeLayout(req.Collection, rp, ttl, types.ToDiskType(req.DiskType)).AddGrowRequestIfAbsent()
}
// With free space but no writable volume and growth already in flight, Assign
// sheds with a retryable ResourceExhausted immediately instead of spinning to
// timeout. ResourceExhausted (not Unavailable) so the shed isn't mistaken for a
// dead channel and doesn't tear down the shared master connection.
func TestAssignShedsLoadWhenGrowthInFlight(t *testing.T) {
ms := newLeaderMaster()
// Free volume slots but no writable volume yet, so growth is the remedy.
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
markGrowthInFlight(t, ms.Topo, req)
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
st, ok := status.FromError(err)
require.True(t, ok)
assert.Equal(t, codes.ResourceExhausted, st.Code())
// Shed immediately rather than spinning out the 10s retry budget.
assert.Less(t, elapsed, 2*time.Second)
// Growth was already pending, so we must not enqueue another grow request.
assert.Len(t, ms.volumeGrowthRequestChan, 0)
}
// The assign that triggers growth must not shed itself: on a cold cluster there
// is no other request to pick up the volume, so it waits for the growth it
// started and returns the fresh volume once it lands.
func TestAssignInitiatorWaitsForItsOwnGrowth(t *testing.T) {
ms := newLeaderMaster()
dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
require.NoError(t, err)
// Simulate growth landing shortly after it is requested by registering a
// writable volume, then draining the request the initiator enqueued.
go func() {
req := <-ms.volumeGrowthRequestChan
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Version: needle.GetCurrentVersion(),
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
ms.Topo.RegisterVolumeLayout(v, dn)
ms.Topo.GetVolumeLayout(req.Option.Collection, rp, needle.EMPTY_TTL, types.ToDiskType("")).DoneGrowRequest()
}()
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.NoError(t, err)
require.NotNil(t, resp)
assert.NotEmpty(t, resp.Fid)
// It waited for growth rather than shedding, but well inside the retry budget.
assert.Less(t, elapsed, 5*time.Second)
}
// An initiator whose growth concludes without yielding a writable volume
// (failed or discarded growth) sheds retryably instead of re-triggering
// growth for the rest of its budget.
func TestAssignInitiatorShedsWhenGrowthConcludesUnfulfilled(t *testing.T) {
ms := newLeaderMaster()
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
require.NoError(t, err)
vl := ms.Topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.ToDiskType(""))
// Growth consumer that fails: clears the flag without registering volumes.
go func() {
<-ms.volumeGrowthRequestChan
vl.DoneGrowRequest()
}()
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
st, ok := status.FromError(err)
require.True(t, ok)
assert.Equal(t, codes.ResourceExhausted, st.Code())
assert.Less(t, elapsed, 2*time.Second)
// Growth was triggered exactly once, not re-enqueued after the failure.
assert.Len(t, ms.volumeGrowthRequestChan, 0)
}
// A location-constrained assign whose location still has room waits for the
// growth it triggered, like an unconstrained one, instead of failing the first
// pick outright.
func TestAssignConstrainedInitiatorWaitsForGrowth(t *testing.T) {
ms := newLeaderMaster()
dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
require.NoError(t, err)
go func() {
req := <-ms.volumeGrowthRequestChan
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Version: needle.GetCurrentVersion(),
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
ms.Topo.RegisterVolumeLayout(v, dn)
ms.Topo.GetVolumeLayout(req.Option.Collection, rp, needle.EMPTY_TTL, types.ToDiskType("")).DoneGrowRequest()
}()
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.NoError(t, err)
require.NotNil(t, resp)
assert.NotEmpty(t, resp.Fid)
assert.Less(t, elapsed, 5*time.Second)
}
// A constrained assign joining a growth already in flight sheds retryably so
// the client retry picks up the fresh volume, instead of failing outright.
func TestAssignConstrainedShedsWhileGrowthPending(t *testing.T) {
ms := newLeaderMaster()
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
markGrowthInFlight(t, ms.Topo, req)
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
st, ok := status.FromError(err)
require.True(t, ok)
assert.Equal(t, codes.ResourceExhausted, st.Code())
assert.Less(t, elapsed, 2*time.Second)
assert.Len(t, ms.volumeGrowthRequestChan, 0)
}
// A constrained initiator whose growth concludes without yielding a writable
// volume sheds retryably so its client's retry can re-trigger growth, instead
// of returning the plain pick error that would stop the retries.
func TestAssignConstrainedShedsWhenGrowthConcludesUnfulfilled(t *testing.T) {
ms := newLeaderMaster()
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
require.NoError(t, err)
vl := ms.Topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.ToDiskType(""))
go func() {
<-ms.volumeGrowthRequestChan
vl.DoneGrowRequest()
}()
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
st, ok := status.FromError(err)
require.True(t, ok)
assert.Equal(t, codes.ResourceExhausted, st.Code())
assert.Less(t, elapsed, 2*time.Second)
assert.Len(t, ms.volumeGrowthRequestChan, 0)
}
// A constrained assign fails fast only when no growth can satisfy the
// constraint: the named location is full (even though another data center has
// room), or volume growth is disabled outright.
func TestAssignConstrainedFailsFastWithoutRoom(t *testing.T) {
tests := []struct {
name string
setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest)
}{
{"location full", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
dn2 := ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2").
GetOrCreateDataNode("127.0.0.2", 8080, 18080, "127.0.0.2", "dn2", map[string]uint32{"": 1})
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
require.NoError(t, err)
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Version: needle.GetCurrentVersion(),
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn2.UpdateVolumes([]storage.VolumeInfo{v})
ms.Topo.RegisterVolumeLayout(v, dn2)
}},
{"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2").
GetOrCreateDataNode("127.0.0.2", 8080, 18080, "127.0.0.2", "dn2", map[string]uint32{"": 100})
ms.option.VolumeGrowthDisabled = true
}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ms := newLeaderMaster()
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh", DataCenter: "dc2"}
tc.setup(t, ms, req)
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
if st, ok := status.FromError(err); ok {
assert.NotEqual(t, codes.Unavailable, st.Code())
assert.NotEqual(t, codes.ResourceExhausted, st.Code())
}
assert.Contains(t, err.Error(), topology.NoWritableVolumes)
assert.Less(t, elapsed, 2*time.Second)
})
}
}
// A cancelled request stops waiting instead of sleeping out the 10s budget.
func TestAssignAbortsOnCancel(t *testing.T) {
ms := newLeaderMaster()
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
// Initiator with its growth never concluding: nobody drains the chan.
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(300 * time.Millisecond)
cancel()
}()
start := time.Now()
_, err := ms.Assign(ctx, req)
elapsed := time.Since(start)
require.ErrorIs(t, err, context.Canceled)
assert.Less(t, elapsed, 2*time.Second)
}
// Out of space, Assign fails fast with the real error rather than masking it as
// a retryable "growth in progress".
func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {
ms := newLeaderMaster()
// One slot, already taken by another collection's volume: capacity is
// registered but genuinely exhausted.
dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 1})
rp, err := super_block.NewReplicaPlacementFromString("000")
require.NoError(t, err)
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Collection: "other",
Version: needle.GetCurrentVersion(),
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
ms.Topo.RegisterVolumeLayout(v, dn)
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
if st, ok := status.FromError(err); ok {
assert.NotEqual(t, codes.Unavailable, st.Code())
assert.NotEqual(t, codes.ResourceExhausted, st.Code())
}
assert.Contains(t, err.Error(), "no free volumes left")
assert.Less(t, elapsed, 2*time.Second)
}
// A topology with no registered capacity is a cluster whose volume servers have
// not heartbeated yet, not one that is full: the first write to a fresh bucket
// races the volume server registration at startup, so Assign must shed with a
// retryable code instead of failing the write outright.
func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) {
ms := newLeaderMaster() // no data nodes: nothing has heartbeated yet
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
st, ok := status.FromError(err)
require.True(t, ok)
assert.Equal(t, codes.ResourceExhausted, st.Code())
assert.Less(t, elapsed, 2*time.Second)
}
// A cluster serving only other media is not one still starting up: capacity for
// the requested disk type will never register, so the startup shed would loop
// until the client's deadline. Assign must fail fast with the real error and
// name the unserved medium — for the growth initiator, for a follower whose
// growth is already in flight, and when growth is disabled outright.
func TestAssignFailsFastWhenDiskTypeUnserved(t *testing.T) {
tests := []struct {
name string
setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest)
}{
{"initiator", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {}},
{"follower joins growth in flight", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
markGrowthInFlight(t, ms.Topo, req)
}},
{"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
ms.option.VolumeGrowthDisabled = true
}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ms := newLeaderMaster()
// ssd capacity registered, but the request asks for the default (hdd).
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1})
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}
tc.setup(t, ms, req)
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
if st, ok := status.FromError(err); ok {
assert.NotEqual(t, codes.Unavailable, st.Code())
assert.NotEqual(t, codes.ResourceExhausted, st.Code())
}
assert.Contains(t, err.Error(), topology.NoWritableVolumes)
assert.Contains(t, err.Error(), `no volume server carries the default (unlabeled) disk layout`)
assert.Less(t, elapsed, 2*time.Second)
})
}
}
// An explicit disk=hdd request is not the unlabeled default: the error must
// name "hdd" so an operator reading it connects it to their hdd configuration.
// ToDiskType folds both "" and "hdd" into HardDriveType, so the formatter must
// look at the original request string, not the canonicalized option.DiskType.
func TestAssignFailsFastNamesExplicitHdd(t *testing.T) {
ms := newLeaderMaster()
// ssd capacity registered, but the request explicitly asks for hdd.
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1})
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh", DiskType: "hdd"}
start := time.Now()
resp, err := ms.Assign(context.Background(), req)
elapsed := time.Since(start)
require.Error(t, err)
require.Nil(t, resp)
assert.Contains(t, err.Error(), topology.NoWritableVolumes)
assert.Contains(t, err.Error(), `no volume server carries the "hdd" disk layout`)
assert.NotContains(t, err.Error(), "default (unlabeled)")
assert.Less(t, elapsed, 2*time.Second)
}
func TestUnservedLayoutWarningBoundedAndExpiring(t *testing.T) {
w := &unservedLayoutWarning{
now: func() time.Time { return time.Unix(0, 0) },
last: make(map[string]time.Time),
}
// Dedupes within the interval.
w.Do("opt", fmt.Errorf("e1"))
w.Do("opt", fmt.Errorf("e2"))
if len(w.last) != 1 {
t.Fatalf("same option logged twice: %v", w.last)
}
// Re-warns after the interval expires.
base := time.Unix(0, 0)
w.now = func() time.Time { return base.Add(unservedLayoutWarnInterval) }
w.Do("opt", fmt.Errorf("e3"))
if len(w.last) != 1 || !w.last["opt"].Equal(base.Add(unservedLayoutWarnInterval)) {
t.Fatalf("expected refreshed timestamp after interval, got %v", w.last)
}
// Hard cap: a client-driven key flood cannot grow the map without bound.
w.now = func() time.Time { return base.Add(2 * unservedLayoutWarnInterval) }
flood := &unservedLayoutWarning{
now: w.now,
last: make(map[string]time.Time),
}
for i := 0; i < unservedLayoutWarnMaxKeys; i++ {
flood.Do(fmt.Sprintf("opt-%d", i), fmt.Errorf("e"))
}
if len(flood.last) != unservedLayoutWarnMaxKeys {
t.Fatalf("expected exactly cap entries, got %d", len(flood.last))
}
flood.Do("opt-flood", fmt.Errorf("e"))
if len(flood.last) != 1 {
t.Fatalf("expected full reset at cap, got %d entries", len(flood.last))
}
}