mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
* master: reject an assign on a follower as Unavailable Assign on a master that is not the raft leader returned raft.NotLeaderError as a plain error, which reaches gRPC clients as codes.Unknown. operation.Assign retries only Unavailable, ResourceExhausted, Canceled and DeadlineExceeded, so every assign made during a leader election failed at once instead of waiting for the new leader. Return it as codes.Unavailable, like the warming-up status in the same file, so clients retry within their 30 s assign budget. The message is unchanged, so callers that match "Not current leader" still work. StreamAssign now answers such a request in-band with the error message instead of ending the stream, as it already does for warm-up. TestAssignOnFollowerIsUnavailable covers it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * pb: keep the cached channel when the peer answers Unavailable with details gRPC's own Unavailable (dial failure, closing transport) never carries details, so one that does came from the peer over a working channel. Closing the shared ClientConn for it cancels every other in-flight RPC on it with "the client connection is closing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: tag the follower's assign rejection with a NOT_LEADER ErrorInfo The detail marks it as the master's answer, so clients retry it without tearing down their cached channel to that master. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
507 lines
19 KiB
Go
507 lines
19 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/raft"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/topology"
|
|
)
|
|
|
|
// leaderRaftServer answers only State(); the embedded nil interface panics if any
|
|
// other method is called.
|
|
type leaderRaftServer struct {
|
|
raft.Server
|
|
}
|
|
|
|
func (leaderRaftServer) State() string { return raft.Leader }
|
|
|
|
func newLeaderMaster() *MasterServer {
|
|
topo := topology.NewTopology("test", sequence.NewMemorySequencer(), 1<<20, 5, false)
|
|
topo.RaftServer = leaderRaftServer{}
|
|
return &MasterServer{
|
|
Topo: topo,
|
|
option: &MasterOption{},
|
|
guard: security.NewGuard(nil, "", 0, "", 0),
|
|
volumeGrowthRequestChan: make(chan *topology.VolumeGrowRequest, 1<<6),
|
|
}
|
|
}
|
|
|
|
// followerRaftServer is a raft follower that knows no leader yet, as during an
|
|
// election.
|
|
type followerRaftServer struct {
|
|
raft.Server
|
|
}
|
|
|
|
func (followerRaftServer) State() string { return raft.Follower }
|
|
func (followerRaftServer) Leader() string { return "" }
|
|
func (followerRaftServer) Name() string { return "127.0.0.1:9333" }
|
|
|
|
func TestAssignOnFollowerIsUnavailable(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.RaftServer = followerRaftServer{}
|
|
|
|
_, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1})
|
|
st, ok := status.FromError(err)
|
|
require.True(t, ok, "want a gRPC status, got %v", err)
|
|
assert.Equal(t, codes.Unavailable, st.Code())
|
|
assert.Equal(t, raft.NotLeaderError.Error(), st.Message())
|
|
require.Len(t, st.Details(), 1)
|
|
info, ok := st.Details()[0].(*errdetails.ErrorInfo)
|
|
require.True(t, ok, "want an ErrorInfo detail, got %T", st.Details()[0])
|
|
assert.Equal(t, "NOT_LEADER", info.GetReason())
|
|
}
|
|
|
|
// markGrowthInFlight flags growth on the same VolumeLayout the handler resolves,
|
|
// by mirroring its exact lookup.
|
|
func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {
|
|
t.Helper()
|
|
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
|
|
require.NoError(t, err)
|
|
ttl, err := needle.ReadTTL(req.Ttl)
|
|
require.NoError(t, err)
|
|
topo.GetVolumeLayout(req.Collection, rp, ttl, types.ToDiskType(req.DiskType)).AddGrowRequestIfAbsent()
|
|
}
|
|
|
|
// With free space but no writable volume and growth already in flight, Assign
|
|
// sheds with a retryable ResourceExhausted immediately instead of spinning to
|
|
// timeout. ResourceExhausted (not Unavailable) so the shed isn't mistaken for a
|
|
// dead channel and doesn't tear down the shared master connection.
|
|
func TestAssignShedsLoadWhenGrowthInFlight(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
// Free volume slots but no writable volume yet, so growth is the remedy.
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
|
|
markGrowthInFlight(t, ms.Topo, req)
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
st, ok := status.FromError(err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, codes.ResourceExhausted, st.Code())
|
|
// Shed immediately rather than spinning out the 10s retry budget.
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
// Growth was already pending, so we must not enqueue another grow request.
|
|
assert.Len(t, ms.volumeGrowthRequestChan, 0)
|
|
}
|
|
|
|
// The assign that triggers growth must not shed itself: on a cold cluster there
|
|
// is no other request to pick up the volume, so it waits for the growth it
|
|
// started and returns the fresh volume once it lands.
|
|
func TestAssignInitiatorWaitsForItsOwnGrowth(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
|
|
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
|
|
require.NoError(t, err)
|
|
|
|
// Simulate growth landing shortly after it is requested by registering a
|
|
// writable volume, then draining the request the initiator enqueued.
|
|
go func() {
|
|
req := <-ms.volumeGrowthRequestChan
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: rp,
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
ms.Topo.RegisterVolumeLayout(v, dn)
|
|
ms.Topo.GetVolumeLayout(req.Option.Collection, rp, needle.EMPTY_TTL, types.ToDiskType("")).DoneGrowRequest()
|
|
}()
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.NoError(t, err)
|
|
require.NotNil(t, resp)
|
|
assert.NotEmpty(t, resp.Fid)
|
|
// It waited for growth rather than shedding, but well inside the retry budget.
|
|
assert.Less(t, elapsed, 5*time.Second)
|
|
}
|
|
|
|
// An initiator whose growth concludes without yielding a writable volume
|
|
// (failed or discarded growth) sheds retryably instead of re-triggering
|
|
// growth for the rest of its budget.
|
|
func TestAssignInitiatorShedsWhenGrowthConcludesUnfulfilled(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
|
|
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
|
|
require.NoError(t, err)
|
|
vl := ms.Topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.ToDiskType(""))
|
|
|
|
// Growth consumer that fails: clears the flag without registering volumes.
|
|
go func() {
|
|
<-ms.volumeGrowthRequestChan
|
|
vl.DoneGrowRequest()
|
|
}()
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
st, ok := status.FromError(err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, codes.ResourceExhausted, st.Code())
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
// Growth was triggered exactly once, not re-enqueued after the failure.
|
|
assert.Len(t, ms.volumeGrowthRequestChan, 0)
|
|
}
|
|
|
|
// A location-constrained assign whose location still has room waits for the
|
|
// growth it triggered, like an unconstrained one, instead of failing the first
|
|
// pick outright.
|
|
func TestAssignConstrainedInitiatorWaitsForGrowth(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
|
|
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
|
|
require.NoError(t, err)
|
|
|
|
go func() {
|
|
req := <-ms.volumeGrowthRequestChan
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: rp,
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
ms.Topo.RegisterVolumeLayout(v, dn)
|
|
ms.Topo.GetVolumeLayout(req.Option.Collection, rp, needle.EMPTY_TTL, types.ToDiskType("")).DoneGrowRequest()
|
|
}()
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.NoError(t, err)
|
|
require.NotNil(t, resp)
|
|
assert.NotEmpty(t, resp.Fid)
|
|
assert.Less(t, elapsed, 5*time.Second)
|
|
}
|
|
|
|
// A constrained assign joining a growth already in flight sheds retryably so
|
|
// the client retry picks up the fresh volume, instead of failing outright.
|
|
func TestAssignConstrainedShedsWhileGrowthPending(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
|
|
markGrowthInFlight(t, ms.Topo, req)
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
st, ok := status.FromError(err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, codes.ResourceExhausted, st.Code())
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
assert.Len(t, ms.volumeGrowthRequestChan, 0)
|
|
}
|
|
|
|
// A constrained initiator whose growth concludes without yielding a writable
|
|
// volume sheds retryably so its client's retry can re-trigger growth, instead
|
|
// of returning the plain pick error that would stop the retries.
|
|
func TestAssignConstrainedShedsWhenGrowthConcludesUnfulfilled(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
|
|
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
|
|
require.NoError(t, err)
|
|
vl := ms.Topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.ToDiskType(""))
|
|
|
|
go func() {
|
|
<-ms.volumeGrowthRequestChan
|
|
vl.DoneGrowRequest()
|
|
}()
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
st, ok := status.FromError(err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, codes.ResourceExhausted, st.Code())
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
assert.Len(t, ms.volumeGrowthRequestChan, 0)
|
|
}
|
|
|
|
// A constrained assign fails fast only when no growth can satisfy the
|
|
// constraint: the named location is full (even though another data center has
|
|
// room), or volume growth is disabled outright.
|
|
func TestAssignConstrainedFailsFastWithoutRoom(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest)
|
|
}{
|
|
{"location full", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
dn2 := ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2").
|
|
GetOrCreateDataNode("127.0.0.2", 8080, 18080, "127.0.0.2", "dn2", map[string]uint32{"": 1})
|
|
rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
|
|
require.NoError(t, err)
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: rp,
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v})
|
|
ms.Topo.RegisterVolumeLayout(v, dn2)
|
|
}},
|
|
{"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
|
|
ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2").
|
|
GetOrCreateDataNode("127.0.0.2", 8080, 18080, "127.0.0.2", "dn2", map[string]uint32{"": 100})
|
|
ms.option.VolumeGrowthDisabled = true
|
|
}},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh", DataCenter: "dc2"}
|
|
tc.setup(t, ms, req)
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
if st, ok := status.FromError(err); ok {
|
|
assert.NotEqual(t, codes.Unavailable, st.Code())
|
|
assert.NotEqual(t, codes.ResourceExhausted, st.Code())
|
|
}
|
|
assert.Contains(t, err.Error(), topology.NoWritableVolumes)
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
})
|
|
}
|
|
}
|
|
|
|
// A cancelled request stops waiting instead of sleeping out the 10s budget.
|
|
func TestAssignAbortsOnCancel(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
|
|
|
|
// Initiator with its growth never concluding: nobody drains the chan.
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
go func() {
|
|
time.Sleep(300 * time.Millisecond)
|
|
cancel()
|
|
}()
|
|
|
|
start := time.Now()
|
|
_, err := ms.Assign(ctx, req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
}
|
|
|
|
// Out of space, Assign fails fast with the real error rather than masking it as
|
|
// a retryable "growth in progress".
|
|
func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
// One slot, already taken by another collection's volume: capacity is
|
|
// registered but genuinely exhausted.
|
|
dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 1})
|
|
rp, err := super_block.NewReplicaPlacementFromString("000")
|
|
require.NoError(t, err)
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Collection: "other",
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: rp,
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
ms.Topo.RegisterVolumeLayout(v, dn)
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
if st, ok := status.FromError(err); ok {
|
|
assert.NotEqual(t, codes.Unavailable, st.Code())
|
|
assert.NotEqual(t, codes.ResourceExhausted, st.Code())
|
|
}
|
|
assert.Contains(t, err.Error(), "no free volumes left")
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
}
|
|
|
|
// A topology with no registered capacity is a cluster whose volume servers have
|
|
// not heartbeated yet, not one that is full: the first write to a fresh bucket
|
|
// races the volume server registration at startup, so Assign must shed with a
|
|
// retryable code instead of failing the write outright.
|
|
func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) {
|
|
ms := newLeaderMaster() // no data nodes: nothing has heartbeated yet
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
st, ok := status.FromError(err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, codes.ResourceExhausted, st.Code())
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
}
|
|
|
|
// A cluster serving only other media is not one still starting up: capacity for
|
|
// the requested disk type will never register, so the startup shed would loop
|
|
// until the client's deadline. Assign must fail fast with the real error and
|
|
// name the unserved medium — for the growth initiator, for a follower whose
|
|
// growth is already in flight, and when growth is disabled outright.
|
|
func TestAssignFailsFastWhenDiskTypeUnserved(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest)
|
|
}{
|
|
{"initiator", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {}},
|
|
{"follower joins growth in flight", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
|
|
markGrowthInFlight(t, ms.Topo, req)
|
|
}},
|
|
{"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
|
|
ms.option.VolumeGrowthDisabled = true
|
|
}},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
// ssd capacity registered, but the request asks for the default (hdd).
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}
|
|
tc.setup(t, ms, req)
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
if st, ok := status.FromError(err); ok {
|
|
assert.NotEqual(t, codes.Unavailable, st.Code())
|
|
assert.NotEqual(t, codes.ResourceExhausted, st.Code())
|
|
}
|
|
assert.Contains(t, err.Error(), topology.NoWritableVolumes)
|
|
assert.Contains(t, err.Error(), `no volume server carries the default (unlabeled) disk layout`)
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
})
|
|
}
|
|
}
|
|
|
|
// An explicit disk=hdd request is not the unlabeled default: the error must
|
|
// name "hdd" so an operator reading it connects it to their hdd configuration.
|
|
// ToDiskType folds both "" and "hdd" into HardDriveType, so the formatter must
|
|
// look at the original request string, not the canonicalized option.DiskType.
|
|
func TestAssignFailsFastNamesExplicitHdd(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
// ssd capacity registered, but the request explicitly asks for hdd.
|
|
ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1})
|
|
|
|
req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh", DiskType: "hdd"}
|
|
|
|
start := time.Now()
|
|
resp, err := ms.Assign(context.Background(), req)
|
|
elapsed := time.Since(start)
|
|
|
|
require.Error(t, err)
|
|
require.Nil(t, resp)
|
|
assert.Contains(t, err.Error(), topology.NoWritableVolumes)
|
|
assert.Contains(t, err.Error(), `no volume server carries the "hdd" disk layout`)
|
|
assert.NotContains(t, err.Error(), "default (unlabeled)")
|
|
assert.Less(t, elapsed, 2*time.Second)
|
|
}
|
|
|
|
func TestUnservedLayoutWarningBoundedAndExpiring(t *testing.T) {
|
|
w := &unservedLayoutWarning{
|
|
now: func() time.Time { return time.Unix(0, 0) },
|
|
last: make(map[string]time.Time),
|
|
}
|
|
|
|
// Dedupes within the interval.
|
|
w.Do("opt", fmt.Errorf("e1"))
|
|
w.Do("opt", fmt.Errorf("e2"))
|
|
if len(w.last) != 1 {
|
|
t.Fatalf("same option logged twice: %v", w.last)
|
|
}
|
|
|
|
// Re-warns after the interval expires.
|
|
base := time.Unix(0, 0)
|
|
w.now = func() time.Time { return base.Add(unservedLayoutWarnInterval) }
|
|
w.Do("opt", fmt.Errorf("e3"))
|
|
if len(w.last) != 1 || !w.last["opt"].Equal(base.Add(unservedLayoutWarnInterval)) {
|
|
t.Fatalf("expected refreshed timestamp after interval, got %v", w.last)
|
|
}
|
|
|
|
// Hard cap: a client-driven key flood cannot grow the map without bound.
|
|
w.now = func() time.Time { return base.Add(2 * unservedLayoutWarnInterval) }
|
|
flood := &unservedLayoutWarning{
|
|
now: w.now,
|
|
last: make(map[string]time.Time),
|
|
}
|
|
for i := 0; i < unservedLayoutWarnMaxKeys; i++ {
|
|
flood.Do(fmt.Sprintf("opt-%d", i), fmt.Errorf("e"))
|
|
}
|
|
if len(flood.last) != unservedLayoutWarnMaxKeys {
|
|
t.Fatalf("expected exactly cap entries, got %d", len(flood.last))
|
|
}
|
|
flood.Do("opt-flood", fmt.Errorf("e"))
|
|
if len(flood.last) != 1 {
|
|
t.Fatalf("expected full reset at cap, got %d entries", len(flood.last))
|
|
}
|
|
}
|