seaweedfs/weed/server/master_grpc_server_assign_test.go

package weed_server

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/seaweedfs/raft"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/security"
	"github.com/seaweedfs/seaweedfs/weed/sequence"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/topology"
)

// leaderRaftServer answers only State(); the embedded nil interface panics if any
// other method is called.
type leaderRaftServer struct {
	raft.Server
}

func (leaderRaftServer) State() string { return raft.Leader }

func newLeaderMaster() *MasterServer {
	topo := topology.NewTopology("test", sequence.NewMemorySequencer(), 1<<20, 5, false)
	topo.RaftServer = leaderRaftServer{}
	return &MasterServer{
		Topo:                    topo,
		option:                  &MasterOption{},
		guard:                   security.NewGuard(nil, "", 0, "", 0),
		volumeGrowthRequestChan: make(chan *topology.VolumeGrowRequest, 1<<6),
	}
}

// markGrowthInFlight flags growth on the same VolumeLayout the handler resolves,
// by mirroring its exact lookup.
func markGrowthInFlight(t *testing.T, topo *topology.Topology, req *master_pb.AssignRequest) {
	t.Helper()
	rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
	require.NoError(t, err)
	ttl, err := needle.ReadTTL(req.Ttl)
	require.NoError(t, err)
	topo.GetVolumeLayout(req.Collection, rp, ttl, types.ToDiskType(req.DiskType)).AddGrowRequestIfAbsent()
}

// With free space but no writable volume and growth already in flight, Assign
// sheds with a retryable ResourceExhausted immediately instead of spinning to
// timeout. ResourceExhausted (not Unavailable) so the shed isn't mistaken for a
// dead channel and doesn't tear down the shared master connection.
func TestAssignShedsLoadWhenGrowthInFlight(t *testing.T) {
	ms := newLeaderMaster()
	// Free volume slots but no writable volume yet, so growth is the remedy.
	ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
	markGrowthInFlight(t, ms.Topo, req)

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	st, ok := status.FromError(err)
	require.True(t, ok)
	assert.Equal(t, codes.ResourceExhausted, st.Code())
	// Shed immediately rather than spinning out the 10s retry budget.
	assert.Less(t, elapsed, 2*time.Second)
	// Growth was already pending, so we must not enqueue another grow request.
	assert.Len(t, ms.volumeGrowthRequestChan, 0)
}

// The assign that triggers growth must not shed itself: on a cold cluster there
// is no other request to pick up the volume, so it waits for the growth it
// started and returns the fresh volume once it lands.
func TestAssignInitiatorWaitsForItsOwnGrowth(t *testing.T) {
	ms := newLeaderMaster()
	dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
	rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
	require.NoError(t, err)

	// Simulate growth landing shortly after it is requested by registering a
	// writable volume, then draining the request the initiator enqueued.
	go func() {
		req := <-ms.volumeGrowthRequestChan
		v := storage.VolumeInfo{
			Id:               needle.VolumeId(1),
			Version:          needle.GetCurrentVersion(),
			ReplicaPlacement: rp,
			Ttl:              needle.EMPTY_TTL,
		}
		dn.UpdateVolumes([]storage.VolumeInfo{v})
		ms.Topo.RegisterVolumeLayout(v, dn)
		ms.Topo.GetVolumeLayout(req.Option.Collection, rp, needle.EMPTY_TTL, types.ToDiskType("")).DoneGrowRequest()
	}()

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.NoError(t, err)
	require.NotNil(t, resp)
	assert.NotEmpty(t, resp.Fid)
	// It waited for growth rather than shedding, but well inside the retry budget.
	assert.Less(t, elapsed, 5*time.Second)
}

// An initiator whose growth concludes without yielding a writable volume
// (failed or discarded growth) sheds retryably instead of re-triggering
// growth for the rest of its budget.
func TestAssignInitiatorShedsWhenGrowthConcludesUnfulfilled(t *testing.T) {
	ms := newLeaderMaster()
	ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
	rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
	require.NoError(t, err)
	vl := ms.Topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.ToDiskType(""))

	// Growth consumer that fails: clears the flag without registering volumes.
	go func() {
		<-ms.volumeGrowthRequestChan
		vl.DoneGrowRequest()
	}()

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	st, ok := status.FromError(err)
	require.True(t, ok)
	assert.Equal(t, codes.ResourceExhausted, st.Code())
	assert.Less(t, elapsed, 2*time.Second)
	// Growth was triggered exactly once, not re-enqueued after the failure.
	assert.Len(t, ms.volumeGrowthRequestChan, 0)
}

// A location-constrained assign whose location still has room waits for the
// growth it triggered, like an unconstrained one, instead of failing the first
// pick outright.
func TestAssignConstrainedInitiatorWaitsForGrowth(t *testing.T) {
	ms := newLeaderMaster()
	dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
	rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
	require.NoError(t, err)

	go func() {
		req := <-ms.volumeGrowthRequestChan
		v := storage.VolumeInfo{
			Id:               needle.VolumeId(1),
			Version:          needle.GetCurrentVersion(),
			ReplicaPlacement: rp,
			Ttl:              needle.EMPTY_TTL,
		}
		dn.UpdateVolumes([]storage.VolumeInfo{v})
		ms.Topo.RegisterVolumeLayout(v, dn)
		ms.Topo.GetVolumeLayout(req.Option.Collection, rp, needle.EMPTY_TTL, types.ToDiskType("")).DoneGrowRequest()
	}()

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.NoError(t, err)
	require.NotNil(t, resp)
	assert.NotEmpty(t, resp.Fid)
	assert.Less(t, elapsed, 5*time.Second)
}

// A constrained assign joining a growth already in flight sheds retryably so
// the client retry picks up the fresh volume, instead of failing outright.
func TestAssignConstrainedShedsWhileGrowthPending(t *testing.T) {
	ms := newLeaderMaster()
	ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
	markGrowthInFlight(t, ms.Topo, req)

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	st, ok := status.FromError(err)
	require.True(t, ok)
	assert.Equal(t, codes.ResourceExhausted, st.Code())
	assert.Less(t, elapsed, 2*time.Second)
	assert.Len(t, ms.volumeGrowthRequestChan, 0)
}

// A constrained initiator whose growth concludes without yielding a writable
// volume sheds retryably so its client's retry can re-trigger growth, instead
// of returning the plain pick error that would stop the retries.
func TestAssignConstrainedShedsWhenGrowthConcludesUnfulfilled(t *testing.T) {
	ms := newLeaderMaster()
	ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000", DataCenter: "dc1"}
	rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
	require.NoError(t, err)
	vl := ms.Topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.ToDiskType(""))

	go func() {
		<-ms.volumeGrowthRequestChan
		vl.DoneGrowRequest()
	}()

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	st, ok := status.FromError(err)
	require.True(t, ok)
	assert.Equal(t, codes.ResourceExhausted, st.Code())
	assert.Less(t, elapsed, 2*time.Second)
	assert.Len(t, ms.volumeGrowthRequestChan, 0)
}

// A constrained assign fails fast only when no growth can satisfy the
// constraint: the named location is full (even though another data center has
// room), or volume growth is disabled outright.
func TestAssignConstrainedFailsFastWithoutRoom(t *testing.T) {
	tests := []struct {
		name  string
		setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest)
	}{
		{"location full", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
			ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
				GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})
			dn2 := ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2").
				GetOrCreateDataNode("127.0.0.2", 8080, 18080, "127.0.0.2", "dn2", map[string]uint32{"": 1})
			rp, err := super_block.NewReplicaPlacementFromString(req.Replication)
			require.NoError(t, err)
			v := storage.VolumeInfo{
				Id:               needle.VolumeId(1),
				Version:          needle.GetCurrentVersion(),
				ReplicaPlacement: rp,
				Ttl:              needle.EMPTY_TTL,
			}
			dn2.UpdateVolumes([]storage.VolumeInfo{v})
			ms.Topo.RegisterVolumeLayout(v, dn2)
		}},
		{"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
			ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2").
				GetOrCreateDataNode("127.0.0.2", 8080, 18080, "127.0.0.2", "dn2", map[string]uint32{"": 100})
			ms.option.VolumeGrowthDisabled = true
		}},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			ms := newLeaderMaster()
			req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh", DataCenter: "dc2"}
			tc.setup(t, ms, req)

			start := time.Now()
			resp, err := ms.Assign(context.Background(), req)
			elapsed := time.Since(start)

			require.Error(t, err)
			require.Nil(t, resp)
			if st, ok := status.FromError(err); ok {
				assert.NotEqual(t, codes.Unavailable, st.Code())
				assert.NotEqual(t, codes.ResourceExhausted, st.Code())
			}
			assert.Contains(t, err.Error(), topology.NoWritableVolumes)
			assert.Less(t, elapsed, 2*time.Second)
		})
	}
}

// A cancelled request stops waiting instead of sleeping out the 10s budget.
func TestAssignAbortsOnCancel(t *testing.T) {
	ms := newLeaderMaster()
	ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 100})

	// Initiator with its growth never concluding: nobody drains the chan.
	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		time.Sleep(300 * time.Millisecond)
		cancel()
	}()

	start := time.Now()
	_, err := ms.Assign(ctx, req)
	elapsed := time.Since(start)

	require.ErrorIs(t, err, context.Canceled)
	assert.Less(t, elapsed, 2*time.Second)
}

// Out of space, Assign fails fast with the real error rather than masking it as
// a retryable "growth in progress".
func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {
	ms := newLeaderMaster()
	// One slot, already taken by another collection's volume: capacity is
	// registered but genuinely exhausted.
	dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 1})
	rp, err := super_block.NewReplicaPlacementFromString("000")
	require.NoError(t, err)
	v := storage.VolumeInfo{
		Id:               needle.VolumeId(1),
		Collection:       "other",
		Version:          needle.GetCurrentVersion(),
		ReplicaPlacement: rp,
		Ttl:              needle.EMPTY_TTL,
	}
	dn.UpdateVolumes([]storage.VolumeInfo{v})
	ms.Topo.RegisterVolumeLayout(v, dn)

	req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	if st, ok := status.FromError(err); ok {
		assert.NotEqual(t, codes.Unavailable, st.Code())
		assert.NotEqual(t, codes.ResourceExhausted, st.Code())
	}
	assert.Contains(t, err.Error(), "no free volumes left")
	assert.Less(t, elapsed, 2*time.Second)
}

// A topology with no registered capacity is a cluster whose volume servers have
// not heartbeated yet, not one that is full: the first write to a fresh bucket
// races the volume server registration at startup, so Assign must shed with a
// retryable code instead of failing the write outright.
func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) {
	ms := newLeaderMaster() // no data nodes: nothing has heartbeated yet

	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	st, ok := status.FromError(err)
	require.True(t, ok)
	assert.Equal(t, codes.ResourceExhausted, st.Code())
	assert.Less(t, elapsed, 2*time.Second)
}

// A cluster serving only other media is not one still starting up: capacity for
// the requested disk type will never register, so the startup shed would loop
// until the client's deadline. Assign must fail fast with the real error and
// name the unserved medium — for the growth initiator, for a follower whose
// growth is already in flight, and when growth is disabled outright.
func TestAssignFailsFastWhenDiskTypeUnserved(t *testing.T) {
	tests := []struct {
		name  string
		setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest)
	}{
		{"initiator", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {}},
		{"follower joins growth in flight", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
			markGrowthInFlight(t, ms.Topo, req)
		}},
		{"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {
			ms.option.VolumeGrowthDisabled = true
		}},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			ms := newLeaderMaster()
			// ssd capacity registered, but the request asks for the default (hdd).
			ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
				GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1})

			req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}
			tc.setup(t, ms, req)

			start := time.Now()
			resp, err := ms.Assign(context.Background(), req)
			elapsed := time.Since(start)

			require.Error(t, err)
			require.Nil(t, resp)
			if st, ok := status.FromError(err); ok {
				assert.NotEqual(t, codes.Unavailable, st.Code())
				assert.NotEqual(t, codes.ResourceExhausted, st.Code())
			}
			assert.Contains(t, err.Error(), topology.NoWritableVolumes)
			assert.Contains(t, err.Error(), `no volume server carries the default (unlabeled) disk layout`)
			assert.Less(t, elapsed, 2*time.Second)
		})
	}
}

// An explicit disk=hdd request is not the unlabeled default: the error must
// name "hdd" so an operator reading it connects it to their hdd configuration.
// ToDiskType folds both "" and "hdd" into HardDriveType, so the formatter must
// look at the original request string, not the canonicalized option.DiskType.
func TestAssignFailsFastNamesExplicitHdd(t *testing.T) {
	ms := newLeaderMaster()
	// ssd capacity registered, but the request explicitly asks for hdd.
	ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1})

	req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh", DiskType: "hdd"}

	start := time.Now()
	resp, err := ms.Assign(context.Background(), req)
	elapsed := time.Since(start)

	require.Error(t, err)
	require.Nil(t, resp)
	assert.Contains(t, err.Error(), topology.NoWritableVolumes)
	assert.Contains(t, err.Error(), `no volume server carries the "hdd" disk layout`)
	assert.NotContains(t, err.Error(), "default (unlabeled)")
	assert.Less(t, elapsed, 2*time.Second)
}

func TestUnservedLayoutWarningBoundedAndExpiring(t *testing.T) {
	w := &unservedLayoutWarning{
		now:  func() time.Time { return time.Unix(0, 0) },
		last: make(map[string]time.Time),
	}

	// Dedupes within the interval.
	w.Do("opt", fmt.Errorf("e1"))
	w.Do("opt", fmt.Errorf("e2"))
	if len(w.last) != 1 {
		t.Fatalf("same option logged twice: %v", w.last)
	}

	// Re-warns after the interval expires.
	base := time.Unix(0, 0)
	w.now = func() time.Time { return base.Add(unservedLayoutWarnInterval) }
	w.Do("opt", fmt.Errorf("e3"))
	if len(w.last) != 1 || !w.last["opt"].Equal(base.Add(unservedLayoutWarnInterval)) {
		t.Fatalf("expected refreshed timestamp after interval, got %v", w.last)
	}

	// Hard cap: a client-driven key flood cannot grow the map without bound.
	w.now = func() time.Time { return base.Add(2 * unservedLayoutWarnInterval) }
	flood := &unservedLayoutWarning{
		now:  w.now,
		last: make(map[string]time.Time),
	}
	for i := 0; i < unservedLayoutWarnMaxKeys; i++ {
		flood.Do(fmt.Sprintf("opt-%d", i), fmt.Errorf("e"))
	}
	if len(flood.last) != unservedLayoutWarnMaxKeys {
		t.Fatalf("expected exactly cap entries, got %d", len(flood.last))
	}
	flood.Do("opt-flood", fmt.Errorf("e"))
	if len(flood.last) != 1 {
		t.Fatalf("expected full reset at cap, got %d entries", len(flood.last))
	}
}