seaweedfs/weed/shell/command_volume_configure_replication_test.go

package shell

import (
	"context"
	"fmt"
	"maps"
	"net"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

type configureReplicationTestVolumeServer struct {
	volume_server_pb.UnimplementedVolumeServerServer
	failure    string
	mu         sync.Mutex
	configured map[uint32]string
}

func (s *configureReplicationTestVolumeServer) VolumeConfigure(_ context.Context, req *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.configured == nil {
		s.configured = make(map[uint32]string)
	}
	s.configured[req.VolumeId] = req.Replication
	return &volume_server_pb.VolumeConfigureResponse{Error: s.failure}, nil
}

// configureReplicationOverlapServer can back several volume servers at once and
// records whether two calls for the same volume were ever in flight together.
type configureReplicationOverlapServer struct {
	volume_server_pb.UnimplementedVolumeServerServer
	mu         sync.Mutex
	inFlight   map[uint32]int
	overlapped bool
}

func (s *configureReplicationOverlapServer) VolumeConfigure(_ context.Context, req *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
	s.mu.Lock()
	s.inFlight[req.VolumeId]++
	if s.inFlight[req.VolumeId] > 1 {
		s.overlapped = true
	}
	s.mu.Unlock()

	time.Sleep(20 * time.Millisecond)

	s.mu.Lock()
	s.inFlight[req.VolumeId]--
	s.mu.Unlock()
	return &volume_server_pb.VolumeConfigureResponse{}, nil
}

func startConfigureReplicationTestVolumeServer(t *testing.T, server volume_server_pb.VolumeServerServer, volumeIds ...uint32) *master_pb.DataNodeInfo {
	t.Helper()
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	srv := grpc.NewServer()
	volume_server_pb.RegisterVolumeServerServer(srv, server)
	go srv.Serve(lis)
	t.Cleanup(srv.Stop)

	port := lis.Addr().(*net.TCPAddr).Port
	disk := &master_pb.DiskInfo{}
	for _, vid := range volumeIds {
		disk.VolumeInfos = append(disk.VolumeInfos, &master_pb.VolumeInformationMessage{Id: vid})
	}
	return &master_pb.DataNodeInfo{
		Id:        fmt.Sprintf("127.0.0.1:%d", port),
		GrpcPort:  uint32(port),
		DiskInfos: map[string]*master_pb.DiskInfo{"": disk},
	}
}

func configureReplicationTestEnv(dataNodes ...*master_pb.DataNodeInfo) (*CommandEnv, *master_pb.TopologyInfo) {
	commandEnv := &CommandEnv{option: &ShellOptions{
		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
	}}
	topologyInfo := &master_pb.TopologyInfo{DataCenterInfos: []*master_pb.DataCenterInfo{{
		RackInfos: []*master_pb.RackInfo{{DataNodeInfos: dataNodes}},
	}}}
	return commandEnv, topologyInfo
}

func allVolumes(*master_pb.VolumeInformationMessage) bool { return true }

func TestConfigureReplicationOnVolumeServersReportsEveryFailingServer(t *testing.T) {
	for _, maxParallelization := range []int{1, DefaultMaxParallelization} {
		t.Run(fmt.Sprintf("maxParallelization=%d", maxParallelization), func(t *testing.T) {
			first := &configureReplicationTestVolumeServer{failure: "first server failed"}
			healthy := &configureReplicationTestVolumeServer{}
			last := &configureReplicationTestVolumeServer{failure: "last server failed"}
			commandEnv, topologyInfo := configureReplicationTestEnv(
				startConfigureReplicationTestVolumeServer(t, first, 1),
				startConfigureReplicationTestVolumeServer(t, healthy, 2, 3),
				startConfigureReplicationTestVolumeServer(t, last, 4),
			)

			err := configureReplicationOnVolumeServers(commandEnv, topologyInfo, allVolumes, "001", maxParallelization)

			if err == nil {
				t.Fatal("expected an error")
			}
			for _, failure := range []string{first.failure, last.failure} {
				if !strings.Contains(err.Error(), failure) {
					t.Errorf("error %q does not mention %q", err, failure)
				}
			}
			healthy.mu.Lock()
			defer healthy.mu.Unlock()
			if expected := map[uint32]string{2: "001", 3: "001"}; !maps.Equal(healthy.configured, expected) {
				t.Errorf("healthy server configured %v, expected %v", healthy.configured, expected)
			}
		})
	}
}

func TestConfigureReplicationOnVolumeServersConfiguresReplicasOneAtATime(t *testing.T) {
	server := &configureReplicationOverlapServer{inFlight: make(map[uint32]int)}
	commandEnv, topologyInfo := configureReplicationTestEnv(
		startConfigureReplicationTestVolumeServer(t, server, 1, 2),
		startConfigureReplicationTestVolumeServer(t, server, 1, 2),
		startConfigureReplicationTestVolumeServer(t, server, 1, 2),
	)

	if err := configureReplicationOnVolumeServers(commandEnv, topologyInfo, allVolumes, "002", DefaultMaxParallelization); err != nil {
		t.Fatal(err)
	}

	server.mu.Lock()
	defer server.mu.Unlock()
	if server.overlapped {
		t.Error("replicas of one volume were configured at the same time")
	}
}

// configureReplicationGateServer blocks inside VolumeConfigure until the test
// releases it, so the test can observe how many servers run at once.
type configureReplicationGateServer struct {
	volume_server_pb.UnimplementedVolumeServerServer
	gate *configureReplicationGate
}

type configureReplicationGate struct {
	entered   chan struct{}
	release   chan struct{}
	mu        sync.Mutex
	active    int
	maxActive int
	calls     int
}

func (s *configureReplicationGateServer) VolumeConfigure(_ context.Context, _ *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
	s.gate.mu.Lock()
	s.gate.calls++
	s.gate.active++
	if s.gate.active > s.gate.maxActive {
		s.gate.maxActive = s.gate.active
	}
	s.gate.mu.Unlock()

	s.gate.entered <- struct{}{}
	<-s.gate.release

	s.gate.mu.Lock()
	s.gate.active--
	s.gate.mu.Unlock()
	return &volume_server_pb.VolumeConfigureResponse{}, nil
}

func TestConfigureReplicationOnVolumeServersRunsServersInParallelUpToTheLimit(t *testing.T) {
	const servers = 4
	const maxParallelization = 2
	gate := &configureReplicationGate{
		entered: make(chan struct{}, servers),
		release: make(chan struct{}),
	}
	var dataNodes []*master_pb.DataNodeInfo
	for i := 0; i < servers; i++ {
		dataNodes = append(dataNodes, startConfigureReplicationTestVolumeServer(t,
			&configureReplicationGateServer{gate: gate}, uint32(i+1)))
	}
	commandEnv, topologyInfo := configureReplicationTestEnv(dataNodes...)

	done := make(chan error, 1)
	go func() {
		done <- configureReplicationOnVolumeServers(commandEnv, topologyInfo, allVolumes, "001", maxParallelization)
	}()

	for i := 0; i < maxParallelization; i++ {
		select {
		case <-gate.entered:
		case <-time.After(10 * time.Second):
			t.Fatalf("only %d servers started; serial execution would hold the second call", i)
		}
	}

	select {
	case <-gate.entered:
		t.Fatalf("more than %d servers ran while calls were held open", maxParallelization)
	case <-time.After(100 * time.Millisecond):
	}

	close(gate.release)
	if err := <-done; err != nil {
		t.Fatal(err)
	}

	gate.mu.Lock()
	defer gate.mu.Unlock()
	if gate.maxActive != maxParallelization {
		t.Errorf("at most %d servers ran together, expected %d", gate.maxActive, maxParallelization)
	}
	if gate.calls != servers {
		t.Errorf("%d servers ran, expected %d", gate.calls, servers)
	}
}