seaweedfs/weed/worker/tasks/ec_balance/detection_test.go

package ec_balance

import (
	"context"
	"net"
	"slices"
	"strings"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
	"github.com/seaweedfs/seaweedfs/weed/worker/types"
)

// The EC balance policy itself is tested in the shared ecbalancer package; these
// tests cover the worker adapter: building the planner topology from the master
// topology (filters, capacity) and the Detection entry point.

func ecTopo(node1Collection string) *master_pb.TopologyInfo {
	node1 := &master_pb.DataNodeInfo{
		Id: "node1",
		DiskInfos: map[string]*master_pb.DiskInfo{
			"": {Type: "", MaxVolumeCount: 100, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
				{Id: 100, Collection: node1Collection, DiskId: 0, EcIndexBits: 0x3FFF}, // 14 shards
			}},
		},
	}
	node2 := &master_pb.DataNodeInfo{
		Id:        "node2",
		DiskInfos: map[string]*master_pb.DiskInfo{"": {Type: "", MaxVolumeCount: 100}},
	}
	return &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{{
			Id: "dc1",
			RackInfos: []*master_pb.RackInfo{
				{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{node1}},
				{Id: "rack2", DataNodeInfos: []*master_pb.DataNodeInfo{node2}},
			},
		}},
	}
}

func TestBuildBalancerTopology(t *testing.T) {
	config := NewDefaultConfig()
	topo, nodeCount, _ := buildBalancerTopology(ecTopo("col1"), config, nil)
	if nodeCount != 2 {
		t.Fatalf("nodeCount = %d, want 2", nodeCount)
	}
	moves := ecbalancer.Plan(topo, ecbalancer.Options{ImbalanceThreshold: 0.01})
	if len(moves) == 0 {
		t.Error("expected cross-rack moves for an all-on-one-rack volume")
	}
}

// TestBuildBalancerTopologyGroupsByHost: two volume servers on host 10.0.0.1
// (different ports) plus three other hosts, a 10+4 volume concentrated on the
// 10.0.0.1 machine. Four machines is enough to spread within parity, so after
// planning the 10.0.0.1 machine must hold <=4 shards of the volume -- which only
// holds if its two ports are grouped into one machine (host wired into the build).
func TestBuildBalancerTopologyGroupsByHost(t *testing.T) {
	mkNode := func(id string, bits uint32) *master_pb.DataNodeInfo {
		di := &master_pb.DiskInfo{Type: "", MaxVolumeCount: 100}
		if bits != 0 {
			di.EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{{Id: 100, Collection: "col1", DiskId: 0, EcIndexBits: bits}}
		}
		return &master_pb.DataNodeInfo{Id: id, DiskInfos: map[string]*master_pb.DiskInfo{"": di}}
	}
	topoInfo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{{
			Id: "dc1",
			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
				mkNode("10.0.0.1:8080", 0x007F), // shards 0-6 on host 10.0.0.1
				mkNode("10.0.0.1:8081", 0x3F80), // shards 7-13 on host 10.0.0.1
				mkNode("10.0.0.2:8080", 0),
				mkNode("10.0.0.3:8080", 0),
				mkNode("10.0.0.4:8080", 0),
			}}},
		}},
	}

	topo, _, _ := buildBalancerTopology(topoInfo, NewDefaultConfig(), nil)
	moves := ecbalancer.Plan(topo, ecbalancer.Options{ImbalanceThreshold: 0.01})

	host := func(nodeID string) string { h, _, _ := net.SplitHostPort(nodeID); return h }
	count := map[string]int{"10.0.0.1": 14}
	for _, m := range moves {
		if m.VolumeID != 100 {
			continue
		}
		count[host(m.SourceNode)]--
		if m.SourceNode != m.TargetNode { // non-dedup move
			count[host(m.TargetNode)]++
		}
	}
	if count["10.0.0.1"] > 4 {
		t.Errorf("machine 10.0.0.1 holds %d shards of the volume after balancing, want <=4 (host grouping not applied)", count["10.0.0.1"])
	}
}

func TestBuildBalancerTopologyCollectionFilter(t *testing.T) {
	config := NewDefaultConfig()
	config.CollectionFilter = "other" // does not match the volume's collection
	allowed, err := wildcard.CompileCollectionMatcher(config.CollectionFilter)
	if err != nil {
		t.Fatalf("CompileCollectionMatcher: %v", err)
	}
	topo, nodeCount, _ := buildBalancerTopology(ecTopo("col1"), config, allowed)
	if nodeCount != 2 {
		t.Fatalf("nodeCount = %d, want 2", nodeCount)
	}
	if moves := ecbalancer.Plan(topo, ecbalancer.Options{ImbalanceThreshold: 0.01}); len(moves) != 0 {
		t.Errorf("filtered-out collection should produce no moves, got %d", len(moves))
	}
}

func TestDetectionDisabled(t *testing.T) {
	config := NewDefaultConfig()
	config.Enabled = false

	results, hasMore, err := Detection(context.Background(), nil, nil, config, 0)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if hasMore {
		t.Error("expected hasMore=false")
	}
	if len(results) != 0 {
		t.Errorf("expected 0 results, got %d", len(results))
	}
}

func TestDetectionNilTopology(t *testing.T) {
	config := NewDefaultConfig()
	clusterInfo := &types.ClusterInfo{ActiveTopology: nil}

	if _, _, err := Detection(context.Background(), nil, clusterInfo, config, 0); err == nil {
		t.Fatal("expected error for nil topology")
	}
}

func TestMovePhasePriority(t *testing.T) {
	cases := map[string]types.TaskPriority{
		"dedup":       types.TaskPriorityHigh,
		"cross_rack":  types.TaskPriorityMedium,
		"within_rack": types.TaskPriorityLow,
		"global":      types.TaskPriorityLow,
	}
	for phase, want := range cases {
		if got := movePhasePriority(phase); got != want {
			t.Errorf("movePhasePriority(%q) = %v, want %v", phase, got, want)
		}
	}
}

// keep the erasure_coding import meaningful for future adapter tests
var _ = erasure_coding.DataShardsCount

// TestSortMovesByPhaseKeepsGlobalAheadOfWithinRack: a long within-rack backlog
// must not push the capped global moves past the detection cap.
func TestSortMovesByPhaseKeepsGlobalAheadOfWithinRack(t *testing.T) {
	moves := []ecbalancer.Move{
		{VolumeID: 1, Phase: "within_rack"},
		{VolumeID: 2, Phase: "dedup"},
		{VolumeID: 3, Phase: "within_rack"},
		{VolumeID: 4, Phase: "global"},
		{VolumeID: 5, Phase: "cross_rack"},
		{VolumeID: 6, Phase: "global"},
	}
	sortMovesByPhase(moves)
	var got []uint32
	for _, m := range moves {
		got = append(got, m.VolumeID)
	}
	if want := []uint32{2, 5, 4, 6, 1, 3}; !slices.Equal(got, want) {
		t.Errorf("order = %v, want %v", got, want)
	}
}

// TestSortMovesByPhaseKeepsAVolumesMovesInPlanOrder: a global move of a shard
// that a within-rack move placed must not run ahead of that move.
func TestSortMovesByPhaseKeepsAVolumesMovesInPlanOrder(t *testing.T) {
	moves := []ecbalancer.Move{
		{VolumeID: 1, ShardID: 3, SourceNode: "within 1", Phase: "within_rack"},
		{VolumeID: 2, SourceNode: "within 2", Phase: "within_rack"},
		{VolumeID: 1, ShardID: 3, SourceNode: "global 1", Phase: "global"},
		{VolumeID: 3, SourceNode: "global 3", Phase: "global"},
		{VolumeID: 2, Collection: "other", SourceNode: "global other 2", Phase: "global"},
	}
	sortMovesByPhase(moves)
	var got []string
	for _, m := range moves {
		got = append(got, m.SourceNode)
	}
	if want := []string{"global 3", "global other 2", "within 1", "within 2", "global 1"}; !slices.Equal(got, want) {
		t.Errorf("order = %q, want %q", got, want)
	}
}

// TestDetectionKeepsGlobalMovesUnderTheCap: the within-rack moves that come
// first in plan order must not push every global move past maxResults.
func TestDetectionKeepsGlobalMovesUnderTheCap(t *testing.T) {
	disk := func(shards ...*master_pb.VolumeEcShardInformationMessage) map[string]*master_pb.DiskInfo {
		return map[string]*master_pb.DiskInfo{"": {Type: "", MaxVolumeCount: 100, EcShardInfos: shards}}
	}
	// Volume 100 sits on one machine (within-rack moves); volumes 1-20 load one
	// server (global moves).
	var loaded []*master_pb.VolumeEcShardInformationMessage
	for vid := uint32(1); vid <= 20; vid++ {
		loaded = append(loaded, &master_pb.VolumeEcShardInformationMessage{Id: vid, Collection: "col1", EcIndexBits: 1})
	}
	topoInfo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{{
			Id: "dc1",
			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
				{Id: "10.0.0.1:8080", DiskInfos: disk(&master_pb.VolumeEcShardInformationMessage{Id: 100, Collection: "col1", EcIndexBits: 0x3FFF})},
				{Id: "10.0.0.2:8080", DiskInfos: disk(loaded...)},
				{Id: "10.0.0.3:8080", DiskInfos: disk()},
				{Id: "10.0.0.4:8080", DiskInfos: disk()},
			}}},
		}},
	}
	at := topology.NewActiveTopology(0)
	if err := at.UpdateTopology(topoInfo); err != nil {
		t.Fatalf("UpdateTopology: %v", err)
	}
	clusterInfo := &types.ClusterInfo{ActiveTopology: at}

	all, _, err := Detection(context.Background(), nil, clusterInfo, NewDefaultConfig(), 0)
	if err != nil {
		t.Fatalf("Detection: %v", err)
	}
	withinRack := 0
	for _, r := range all {
		if strings.Contains(r.Reason, " within_rack:") {
			withinRack++
		}
	}
	if withinRack < 2 {
		t.Fatalf("fixture plans %d within-rack moves, want at least 2", withinRack)
	}

	results, hasMore, err := Detection(context.Background(), nil, clusterInfo, NewDefaultConfig(), 1)
	if err != nil {
		t.Fatalf("Detection: %v", err)
	}
	if len(results) != 1 || !hasMore {
		t.Fatalf("got %d results (hasMore %v), want 1 and more", len(results), hasMore)
	}
	if !strings.Contains(results[0].Reason, " global:") {
		t.Errorf("kept %q, want a global move", results[0].Reason)
	}
}

// TestDetectionUsesConfiguredGlobalCap: the configured per-rack cap reaches the planner, and
// an unset one plans the default rather than the planner's unlimited 0.
func TestDetectionUsesConfiguredGlobalCap(t *testing.T) {
	disk := func(shards ...*master_pb.VolumeEcShardInformationMessage) map[string]*master_pb.DiskInfo {
		return map[string]*master_pb.DiskInfo{"": {Type: "", MaxVolumeCount: 100, EcShardInfos: shards}}
	}
	var loaded []*master_pb.VolumeEcShardInformationMessage
	for vid := uint32(1); vid <= 40; vid++ {
		loaded = append(loaded, &master_pb.VolumeEcShardInformationMessage{Id: vid, Collection: "col1", EcIndexBits: 1})
	}
	topoInfo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{{
			Id: "dc1",
			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
				{Id: "10.0.0.1:8080", DiskInfos: disk(loaded...)},
				{Id: "10.0.0.2:8080", DiskInfos: disk()},
				{Id: "10.0.0.3:8080", DiskInfos: disk()},
				{Id: "10.0.0.4:8080", DiskInfos: disk()},
			}}},
		}},
	}
	at := topology.NewActiveTopology(0)
	if err := at.UpdateTopology(topoInfo); err != nil {
		t.Fatalf("UpdateTopology: %v", err)
	}
	clusterInfo := &types.ClusterInfo{ActiveTopology: at}

	for _, tc := range []struct{ configured, maxResults, want int }{
		{0, 0, DefaultGlobalMaxMovesPerRack},
		{25, 0, 25},
		{25, 20, 20},
	} {
		config := NewDefaultConfig()
		config.GlobalMaxMovesPerRack = tc.configured
		results, _, err := Detection(context.Background(), nil, clusterInfo, config, tc.maxResults)
		if err != nil {
			t.Fatalf("Detection: %v", err)
		}
		global := 0
		for _, r := range results {
			if strings.Contains(r.Reason, " global:") {
				global++
			}
		}
		if global != tc.want {
			t.Errorf("cap %d with %d results planned %d global moves, want %d", tc.configured, tc.maxResults, global, tc.want)
		}
	}
}

// TestGlobalMaxMovesPerRackBoundedByMaxResults: a rack plans no more global moves than the
// detection can keep.
func TestGlobalMaxMovesPerRackBoundedByMaxResults(t *testing.T) {
	config := &Config{GlobalMaxMovesPerRack: 5000}
	if got := globalMaxMovesPerRack(config, 500); got != 500 {
		t.Errorf("cap with 500 results = %d, want 500", got)
	}
	if got := globalMaxMovesPerRack(config, 0); got != 5000 {
		t.Errorf("cap with unlimited results = %d, want 5000", got)
	}
}