seaweedfs/weed/server/block_recovery_adversarial_test.go

package weed_server

import (
	"sync/atomic"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)

// ============================================================
// Phase 09 P4: Adversarial tests for RecoveryManager
// ============================================================

// --- Adversarial 1: Rapid triple supersede ---

func TestAdversarial_P4_RapidTripleSupersede(t *testing.T) {
	bs, volPath := createTestBlockServiceWithVol(t)
	rm := bs.v2Recovery

	// Write data so recovery has work.
	bs.blockStore.WithVolume(volPath, func(vol *blockvol.BlockVol) error {
		for i := 0; i < 5; i++ {
			vol.WriteLBA(uint64(i), make([]byte, 4096))
		}
		return nil
	})

	var executionCount atomic.Int32

	rm.OnBeforeExecute = func(replicaID string) {
		executionCount.Add(1)
		// Simulate slow execution — each goroutine takes 100ms.
		time.Sleep(100 * time.Millisecond)
	}

	makeAssignment := func(epoch uint64) []blockvol.BlockVolumeAssignment {
		return []blockvol.BlockVolumeAssignment{
			{Path: volPath, Epoch: epoch, Role: uint32(blockvol.RolePrimary),
				ReplicaServerID: "vs2", ReplicaDataAddr: "10.0.0.2:9333", ReplicaCtrlAddr: "10.0.0.2:9334"},
		}
	}

	// Rapid fire: epoch 1, 2, 3 in quick succession.
	// Each ProcessAssignments must drain the previous before starting the next.
	bs.ProcessAssignments(makeAssignment(1))
	time.Sleep(20 * time.Millisecond) // let goroutine start

	bs.ProcessAssignments(makeAssignment(2))
	bs.ProcessAssignments(makeAssignment(3))

	// Wait for all to settle.
	time.Sleep(500 * time.Millisecond)

	// Must have at most 1 active task (the final one, or 0 if it completed).
	count := rm.ActiveTaskCount()
	if count > 1 {
		t.Fatalf("goroutine leak: %d active tasks after triple supersede", count)
	}

	t.Logf("adversarial 1: triple supersede — %d executions, %d active tasks, no leak",
		executionCount.Load(), count)
}

// --- Adversarial 2: Shutdown during slow goroutine (not permanently stuck) ---

func TestAdversarial_P4_ShutdownDuringSlow(t *testing.T) {
	bs, volPath := createTestBlockServiceWithVol(t)
	rm := bs.v2Recovery

	bs.blockStore.WithVolume(volPath, func(vol *blockvol.BlockVol) error {
		for i := 0; i < 5; i++ {
			vol.WriteLBA(uint64(i), make([]byte, 4096))
		}
		return nil
	})

	started := make(chan struct{}, 1)
	rm.OnBeforeExecute = func(replicaID string) {
		started <- struct{}{}
		// Simulate slow execution — 500ms, not permanently stuck.
		time.Sleep(500 * time.Millisecond)
	}

	bs.ProcessAssignments([]blockvol.BlockVolumeAssignment{
		{Path: volPath, Epoch: 1, Role: uint32(blockvol.RolePrimary),
			ReplicaServerID: "vs2", ReplicaDataAddr: "10.0.0.2:9333", ReplicaCtrlAddr: "10.0.0.2:9334"},
	})

	select {
	case <-started:
	case <-time.After(5 * time.Second):
		t.Fatal("goroutine did not start")
	}

	// Shutdown while goroutine is in slow execution.
	done := make(chan bool, 1)
	go func() {
		rm.Shutdown()
		done <- true
	}()

	select {
	case <-done:
		t.Log("adversarial 2: Shutdown completed after slow goroutine finished")
	case <-time.After(5 * time.Second):
		t.Fatal("Shutdown did not complete within 5 seconds")
	}

	if rm.ActiveTaskCount() != 0 {
		t.Fatalf("tasks after shutdown: %d", rm.ActiveTaskCount())
	}
}

// --- Adversarial 3: Two replicas, no cross-interference ---

func TestAdversarial_P4_TwoReplicas_NoInterference(t *testing.T) {
	bs, volPath := createTestBlockServiceWithVol(t)
	rm := bs.v2Recovery

	bs.blockStore.WithVolume(volPath, func(vol *blockvol.BlockVol) error {
		for i := 0; i < 5; i++ {
			vol.WriteLBA(uint64(i), make([]byte, 4096))
		}
		return nil
	})

	var execIDs []string
	var execMu = make(chan struct{}, 1)
	rm.OnBeforeExecute = func(replicaID string) {
		execMu <- struct{}{}
		execIDs = append(execIDs, replicaID)
		<-execMu
	}

	// Two different replicas in one assignment batch.
	bs.ProcessAssignments([]blockvol.BlockVolumeAssignment{
		{Path: volPath, Epoch: 1, Role: uint32(blockvol.RolePrimary),
			ReplicaServerID: "vs2", ReplicaDataAddr: "10.0.0.2:9333", ReplicaCtrlAddr: "10.0.0.2:9334",
			ReplicaAddrs: []blockvol.ReplicaAddr{
				{DataAddr: "10.0.0.2:9333", CtrlAddr: "10.0.0.2:9334", ServerID: "vs2"},
				{DataAddr: "10.0.0.3:9333", CtrlAddr: "10.0.0.3:9334", ServerID: "vs3"},
			},
		},
	})

	time.Sleep(300 * time.Millisecond)

	// Both replicas should have gotten recovery goroutines.
	// No cross-interference: each has its own task with its own done channel.
	replicaID1 := volPath + "/vs2"
	replicaID2 := volPath + "/vs3"

	s1 := bs.v2Orchestrator.Registry.Sender(replicaID1)
	s2 := bs.v2Orchestrator.Registry.Sender(replicaID2)

	if s1 == nil {
		t.Fatal("sender for vs2 not created")
	}
	if s2 == nil {
		t.Fatal("sender for vs3 not created")
	}

	t.Logf("adversarial 3: two replicas — vs2=%s vs3=%s, no cross-interference",
		s1.State(), s2.State())
}