mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
Ping QiuandClaude Opus 4.6
a9a5e455c6
Lookup() and ListAll() now return value copies (not pointers to internal registry state). Callers can no longer mutate registry entries without holding a lock. Added clone() on BlockVolumeEntry with deep-copied Replicas slice. Added UpdateEntry(name, func(*BlockVolumeEntry)) for locked mutation. ListByServer() also returns copies. Migrated 1 production mutation (ReplicaPlacement + Preset in create handler) and ~20 test mutations to use UpdateEntry. 5 new copy-correctness tests: Lookup returns copy, Replicas slice isolated, ListAll returns copies, UpdateEntry mutates, UpdateEntry not-found error. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
888 lines
31 KiB
Go
888 lines
31 KiB
Go
package weed_server
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
||
)
|
||
|
||
// ============================================================
|
||
// CP8-2 Adversarial Test Suite
|
||
//
|
||
// 12 scenarios covering multi-replica, scrub, promotion gating,
|
||
// heartbeat reconciliation, and partial-failure edge cases.
|
||
// ============================================================
|
||
|
||
// qaCP82Master creates a MasterServer with 3 block-capable servers
|
||
// and configurable mocks for adversarial testing.
|
||
func qaCP82Master(t *testing.T) *MasterServer {
|
||
t.Helper()
|
||
ms := &MasterServer{
|
||
blockRegistry: NewBlockVolumeRegistry(),
|
||
blockAssignmentQueue: NewBlockAssignmentQueue(),
|
||
blockFailover: newBlockFailoverState(),
|
||
}
|
||
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
|
||
return &blockAllocResult{
|
||
Path: fmt.Sprintf("/data/%s.blk", name),
|
||
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
|
||
ISCSIAddr: server + ":3260",
|
||
ReplicaDataAddr: server + ":14260",
|
||
ReplicaCtrlAddr: server + ":14261",
|
||
RebuildListenAddr: server + ":15000",
|
||
}, nil
|
||
}
|
||
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
|
||
return nil
|
||
}
|
||
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
|
||
return newSize, nil
|
||
}
|
||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error {
|
||
return nil
|
||
}
|
||
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) {
|
||
return 2 << 30, nil
|
||
}
|
||
ms.blockVSCancelExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) error {
|
||
return nil
|
||
}
|
||
ms.blockRegistry.MarkBlockCapable("vs1:9333")
|
||
ms.blockRegistry.MarkBlockCapable("vs2:9333")
|
||
ms.blockRegistry.MarkBlockCapable("vs3:9333")
|
||
return ms
|
||
}
|
||
|
||
// qaRegisterRF2WithLSN creates an RF=2 volume with explicit WAL LSN state.
|
||
func qaRegisterRF2WithLSN(t *testing.T, ms *MasterServer, name, primary, replica string, epoch, primaryLSN, replicaLSN uint64) {
|
||
t.Helper()
|
||
entry := &BlockVolumeEntry{
|
||
Name: name,
|
||
VolumeServer: primary,
|
||
Path: fmt.Sprintf("/data/%s.blk", name),
|
||
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
|
||
ISCSIAddr: primary + ":3260",
|
||
SizeBytes: 1 << 30,
|
||
Epoch: epoch,
|
||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||
Status: StatusActive,
|
||
WALHeadLSN: primaryLSN,
|
||
LeaseTTL: 5 * time.Second,
|
||
LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired
|
||
ReplicaServer: replica,
|
||
ReplicaPath: fmt.Sprintf("/data/%s.blk", name),
|
||
ReplicaIQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
|
||
ReplicaISCSIAddr: replica + ":3260",
|
||
Replicas: []ReplicaInfo{
|
||
{
|
||
Server: replica,
|
||
Path: fmt.Sprintf("/data/%s.blk", name),
|
||
IQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
|
||
ISCSIAddr: replica + ":3260",
|
||
HealthScore: 1.0,
|
||
WALHeadLSN: replicaLSN,
|
||
LastHeartbeat: time.Now(), // fresh
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
},
|
||
},
|
||
}
|
||
if err := ms.blockRegistry.Register(entry); err != nil {
|
||
t.Fatalf("register %s: %v", name, err)
|
||
}
|
||
}
|
||
|
||
// qaRegisterRF3WithState creates an RF=3 volume with per-replica state control.
|
||
func qaRegisterRF3WithState(t *testing.T, ms *MasterServer, name, primary string,
|
||
replicas []struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
},
|
||
primaryLSN uint64,
|
||
) {
|
||
t.Helper()
|
||
entry := &BlockVolumeEntry{
|
||
Name: name,
|
||
VolumeServer: primary,
|
||
Path: fmt.Sprintf("/data/%s.blk", name),
|
||
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
|
||
ISCSIAddr: primary + ":3260",
|
||
SizeBytes: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||
Status: StatusActive,
|
||
WALHeadLSN: primaryLSN,
|
||
ReplicaFactor: len(replicas) + 1,
|
||
LeaseTTL: 5 * time.Second,
|
||
LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired
|
||
}
|
||
for i, r := range replicas {
|
||
ri := ReplicaInfo{
|
||
Server: r.Server,
|
||
Path: fmt.Sprintf("/data/%s.blk", name),
|
||
IQN: fmt.Sprintf("iqn.2024.test:%s-r%d", name, i+1),
|
||
ISCSIAddr: r.Server + ":3260",
|
||
HealthScore: r.HealthScore,
|
||
WALHeadLSN: r.WALHeadLSN,
|
||
Role: blockvol.RoleToWire(r.Role),
|
||
LastHeartbeat: r.Heartbeat,
|
||
}
|
||
entry.Replicas = append(entry.Replicas, ri)
|
||
}
|
||
// Sync scalar fields from first replica.
|
||
if len(entry.Replicas) > 0 {
|
||
entry.ReplicaServer = entry.Replicas[0].Server
|
||
entry.ReplicaPath = entry.Replicas[0].Path
|
||
entry.ReplicaIQN = entry.Replicas[0].IQN
|
||
entry.ReplicaISCSIAddr = entry.Replicas[0].ISCSIAddr
|
||
}
|
||
if err := ms.blockRegistry.Register(entry); err != nil {
|
||
t.Fatalf("register %s: %v", name, err)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-1: PrimaryCrash_AfterAck_BeforeReplicaBarrier
|
||
//
|
||
// Verify failover data loss window is bounded by barrier_lag_lsn.
|
||
// Simulates primary at LSN 200, replica at barrier LSN 190.
|
||
// After failover, new primary should be at replica's LSN (190).
|
||
// The gap (10 LSN entries) is the bounded loss window.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_PrimaryCrash_AfterAck_BeforeReplicaBarrier(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
qaRegisterRF2WithLSN(t, ms, "vol-lag", "vs1:9333", "vs2:9333", 1, 200, 190)
|
||
|
||
// Kill primary.
|
||
ms.failoverBlockVolumes("vs1:9333")
|
||
|
||
entry, ok := ms.blockRegistry.Lookup("vol-lag")
|
||
if !ok {
|
||
t.Fatal("volume should still exist after failover")
|
||
}
|
||
if entry.VolumeServer != "vs2:9333" {
|
||
t.Fatalf("replica should be promoted, got %q", entry.VolumeServer)
|
||
}
|
||
// The promoted replica was at LSN 190. The 10-LSN gap represents
|
||
// acknowledged-but-unbarriered writes (bounded data loss).
|
||
// New epoch should be 2.
|
||
if entry.Epoch != 2 {
|
||
t.Fatalf("epoch: got %d, want 2", entry.Epoch)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-2: ReplicaHeartbeatSpoof_DoesNotDeletePrimary
|
||
//
|
||
// Regression test for Fix #1. A replica heartbeat that does NOT
|
||
// include the primary path must NOT delete the volume entry.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_ReplicaHeartbeatSpoof_DoesNotDeletePrimary(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
qaRegisterRF2WithLSN(t, ms, "vol-spoof", "vs1:9333", "vs2:9333", 1, 100, 100)
|
||
|
||
// Simulate replica heartbeat from vs2 with ONLY its replica path.
|
||
// This must NOT remove the volume (vs1 is the primary, not vs2).
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-spoof.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
},
|
||
}, "")
|
||
|
||
entry, ok := ms.blockRegistry.Lookup("vol-spoof")
|
||
if !ok {
|
||
t.Fatal("volume must NOT be deleted by replica heartbeat (Fix #1 regression)")
|
||
}
|
||
if entry.VolumeServer != "vs1:9333" {
|
||
t.Fatalf("primary must remain vs1:9333, got %q", entry.VolumeServer)
|
||
}
|
||
|
||
// Now simulate a full heartbeat from vs2 with NO volumes at all.
|
||
// This should remove vs2 as replica but NOT delete the volume.
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{}, "")
|
||
|
||
entry, ok = ms.blockRegistry.Lookup("vol-spoof")
|
||
if !ok {
|
||
t.Fatal("volume must still exist after empty replica heartbeat")
|
||
}
|
||
if entry.VolumeServer != "vs1:9333" {
|
||
t.Fatal("primary must still be vs1:9333")
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-3: PromotionRejects_StaleButHealthyReplica
|
||
//
|
||
// Replica has perfect health (1.0) but stale heartbeat.
|
||
// Must NOT be promoted (Gate 1: heartbeat freshness).
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_PromotionRejects_StaleButHealthyReplica(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
staleTime := time.Now().Add(-5 * time.Minute) // way beyond 2×LeaseTTL
|
||
qaRegisterRF3WithState(t, ms, "vol-stale", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, staleTime}, // healthy but stale
|
||
{"vs3:9333", 0.5, 100, blockvol.RoleReplica, staleTime}, // also stale
|
||
},
|
||
100,
|
||
)
|
||
|
||
_, err := ms.blockRegistry.PromoteBestReplica("vol-stale")
|
||
if err == nil {
|
||
t.Fatal("expected promotion to fail — all replicas have stale heartbeats")
|
||
}
|
||
// Volume should remain unchanged.
|
||
entry, _ := ms.blockRegistry.Lookup("vol-stale")
|
||
if entry.VolumeServer != "vs1:9333" {
|
||
t.Fatalf("primary must not change, got %q", entry.VolumeServer)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-4: PromotionRejects_RebuildingReplica
|
||
//
|
||
// Replica is fresh and healthy but in RoleRebuilding.
|
||
// Must NOT be promoted (Gate 3: role check).
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_PromotionRejects_RebuildingReplica(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
now := time.Now()
|
||
qaRegisterRF3WithState(t, ms, "vol-rebuild", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // rebuilding
|
||
{"vs3:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // also rebuilding
|
||
},
|
||
100,
|
||
)
|
||
|
||
_, err := ms.blockRegistry.PromoteBestReplica("vol-rebuild")
|
||
if err == nil {
|
||
t.Fatal("expected promotion to fail — all replicas are rebuilding")
|
||
}
|
||
|
||
// One rebuilding, one ready: only the ready one should be promoted.
|
||
ms2 := qaCP82Master(t)
|
||
qaRegisterRF3WithState(t, ms2, "vol-mixed", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now},
|
||
{"vs3:9333", 0.7, 100, blockvol.RoleReplica, now}, // eligible despite lower health
|
||
},
|
||
100,
|
||
)
|
||
|
||
_, err = ms2.blockRegistry.PromoteBestReplica("vol-mixed")
|
||
if err != nil {
|
||
t.Fatalf("expected promotion to succeed: %v", err)
|
||
}
|
||
entry, _ := ms2.blockRegistry.Lookup("vol-mixed")
|
||
if entry.VolumeServer != "vs3:9333" {
|
||
t.Fatalf("vs3 should be promoted (only eligible), got %q", entry.VolumeServer)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-5: PromotionToleranceBoundary_ExactLSN
|
||
//
|
||
// Tests promotion_lsn_tolerance boundary: tolerance-1, exact, tolerance+1.
|
||
// Catches off-by-one in the eligibility gate.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_PromotionToleranceBoundary_ExactLSN(t *testing.T) {
|
||
const tolerance uint64 = 50
|
||
const primaryLSN uint64 = 200
|
||
|
||
cases := []struct {
|
||
name string
|
||
replicaLSN uint64
|
||
wantErr bool
|
||
}{
|
||
// replicaLSN + tolerance < primaryLSN → ineligible
|
||
// 149 + 50 = 199 < 200 → INELIGIBLE
|
||
{"below_tolerance", primaryLSN - tolerance - 1, true},
|
||
// 150 + 50 = 200 == 200 → ELIGIBLE (not strict less-than)
|
||
{"at_tolerance", primaryLSN - tolerance, false},
|
||
// 151 + 50 = 201 > 200 → ELIGIBLE
|
||
{"above_tolerance", primaryLSN - tolerance + 1, false},
|
||
}
|
||
|
||
for _, tc := range cases {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
ms.blockRegistry.SetPromotionLSNTolerance(tolerance)
|
||
|
||
now := time.Now()
|
||
qaRegisterRF3WithState(t, ms, "vol-tol", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, tc.replicaLSN, blockvol.RoleReplica, now},
|
||
},
|
||
primaryLSN,
|
||
)
|
||
|
||
_, err := ms.blockRegistry.PromoteBestReplica("vol-tol")
|
||
if tc.wantErr && err == nil {
|
||
t.Fatalf("replicaLSN=%d: expected promotion to fail", tc.replicaLSN)
|
||
}
|
||
if !tc.wantErr && err != nil {
|
||
t.Fatalf("replicaLSN=%d: expected promotion to succeed: %v", tc.replicaLSN, err)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-6: MasterRestart_ReconstructReplicas_ThenFailover
|
||
//
|
||
// Simulates master restart: empty registry, full heartbeats
|
||
// from primary and replica servers reconstruct state, then
|
||
// failover correctly promotes the replica.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_MasterRestart_ReconstructReplicas_ThenFailover(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
|
||
// Phase 1: Primary heartbeat — auto-registers volume from heartbeat.
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-restart.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 5,
|
||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||
WalHeadLsn: 500,
|
||
},
|
||
}, "")
|
||
|
||
entry, ok := ms.blockRegistry.Lookup("vol-restart")
|
||
if !ok {
|
||
t.Fatal("volume should be auto-registered from primary heartbeat")
|
||
}
|
||
if entry.VolumeServer != "vs1:9333" {
|
||
t.Fatalf("primary: got %q, want vs1:9333", entry.VolumeServer)
|
||
}
|
||
|
||
// Phase 2: Replica heartbeat — should reconstruct ReplicaInfo.
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-restart.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 5,
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
WalHeadLsn: 498,
|
||
},
|
||
}, "")
|
||
|
||
entry, _ = ms.blockRegistry.Lookup("vol-restart")
|
||
if len(entry.Replicas) == 0 {
|
||
t.Fatal("replica should be reconstructed from heartbeat (Fix #3)")
|
||
}
|
||
if entry.Replicas[0].Server != "vs2:9333" {
|
||
t.Fatalf("replica server: got %q, want vs2:9333", entry.Replicas[0].Server)
|
||
}
|
||
|
||
// Phase 3: Set lease expired and trigger failover.
|
||
ms.blockRegistry.UpdateEntry("vol-restart", func(e *BlockVolumeEntry) {
|
||
e.LeaseTTL = 5 * time.Second
|
||
e.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
|
||
// Ensure replica is eligible for promotion.
|
||
e.Replicas[0].LastHeartbeat = time.Now()
|
||
e.Replicas[0].Role = blockvol.RoleToWire(blockvol.RoleReplica)
|
||
})
|
||
|
||
ms.failoverBlockVolumes("vs1:9333")
|
||
|
||
entry, ok = ms.blockRegistry.Lookup("vol-restart")
|
||
if !ok {
|
||
t.Fatal("volume should survive failover")
|
||
}
|
||
if entry.VolumeServer != "vs2:9333" {
|
||
t.Fatalf("after failover: primary should be vs2:9333, got %q", entry.VolumeServer)
|
||
}
|
||
if entry.Epoch <= 5 {
|
||
t.Fatalf("epoch should be bumped, got %d", entry.Epoch)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-7: RF3_OneReplicaFlaps_UnderWriteLoad
|
||
//
|
||
// One replica rapidly disconnects/reconnects while concurrent
|
||
// failover checks run. Must not cause split-brain or panic.
|
||
// Simulates via concurrent UpdateFullHeartbeat + failoverBlockVolumes.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_RF3_OneReplicaFlaps_UnderWriteLoad(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
now := time.Now()
|
||
qaRegisterRF3WithState(t, ms, "vol-flap", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, now},
|
||
{"vs3:9333", 1.0, 100, blockvol.RoleReplica, now},
|
||
},
|
||
100,
|
||
)
|
||
|
||
var wg sync.WaitGroup
|
||
const rounds = 50
|
||
|
||
// Goroutine 1: flapping replica vs3 (heartbeat with/without volume).
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for i := 0; i < rounds; i++ {
|
||
if i%2 == 0 {
|
||
// Replica reports volume.
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-flap.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
WalHeadLsn: 100,
|
||
},
|
||
}, "")
|
||
} else {
|
||
// Replica reports no volumes (simulates disconnect).
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{}, "")
|
||
}
|
||
}
|
||
}()
|
||
|
||
// Goroutine 2: concurrent heartbeats from primary.
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for i := 0; i < rounds; i++ {
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-flap.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||
WalHeadLsn: uint64(100 + i),
|
||
},
|
||
}, "")
|
||
}
|
||
}()
|
||
|
||
// Goroutine 3: concurrent heartbeats from stable replica vs2.
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for i := 0; i < rounds; i++ {
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-flap.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
WalHeadLsn: uint64(100 + i),
|
||
},
|
||
}, "")
|
||
}
|
||
}()
|
||
|
||
wg.Wait()
|
||
|
||
// Invariant: volume must still exist, primary must be vs1.
|
||
entry, ok := ms.blockRegistry.Lookup("vol-flap")
|
||
if !ok {
|
||
t.Fatal("volume must survive flapping replica")
|
||
}
|
||
if entry.VolumeServer != "vs1:9333" {
|
||
t.Fatalf("primary must remain vs1:9333, got %q (split-brain!)", entry.VolumeServer)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-8: AssignmentPrecedence_ReplicaAddrsVsScalar
|
||
//
|
||
// Creates RF=3 volume and verifies primary assignment includes
|
||
// ReplicaAddrs (not just scalar fields). VS should use
|
||
// ReplicaAddrs when non-empty.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_AssignmentPrecedence_ReplicaAddrsVsScalar(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
ctx := context.Background()
|
||
|
||
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
|
||
Name: "vol-addrs",
|
||
SizeBytes: 1 << 30,
|
||
ReplicaFactor: 3,
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("create: %v", err)
|
||
}
|
||
|
||
primary := resp.VolumeServer
|
||
assignments := ms.blockAssignmentQueue.Peek(primary)
|
||
|
||
// Find the primary assignment.
|
||
var primaryAssignment *blockvol.BlockVolumeAssignment
|
||
for i := range assignments {
|
||
if blockvol.RoleFromWire(assignments[i].Role) == blockvol.RolePrimary {
|
||
primaryAssignment = &assignments[i]
|
||
break
|
||
}
|
||
}
|
||
|
||
if primaryAssignment == nil {
|
||
t.Fatal("no primary assignment found")
|
||
}
|
||
|
||
// RF=3 → 2 replicas → ReplicaAddrs should have 2 entries.
|
||
if len(primaryAssignment.ReplicaAddrs) != 2 {
|
||
t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", len(primaryAssignment.ReplicaAddrs))
|
||
}
|
||
|
||
// Verify each ReplicaAddr has data+ctrl.
|
||
for i, ra := range primaryAssignment.ReplicaAddrs {
|
||
if ra.DataAddr == "" || ra.CtrlAddr == "" {
|
||
t.Errorf("ReplicaAddrs[%d]: data=%q ctrl=%q — both must be non-empty", i, ra.DataAddr, ra.CtrlAddr)
|
||
}
|
||
}
|
||
|
||
// For RF=3, legacy scalar ReplicaDataAddr is NOT set on the primary
|
||
// assignment (only ReplicaAddrs is used). Verify precedence: ReplicaAddrs
|
||
// is the authoritative source when non-empty.
|
||
if len(primaryAssignment.ReplicaAddrs) > 0 && primaryAssignment.ReplicaDataAddr != "" {
|
||
t.Error("legacy ReplicaDataAddr should be empty when ReplicaAddrs is populated (RF=3)")
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-9: ScrubConcurrentWrites_NoFalseCorruption
|
||
//
|
||
// Heavy writes while scrub runs. Expect zero false positives.
|
||
// Tests at the HealthScore level (engine scrub tested separately).
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_ScrubConcurrentWrites_NoFalseCorruption(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
now := time.Now()
|
||
qaRegisterRF3WithState(t, ms, "vol-scrub", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, 500, blockvol.RoleReplica, now},
|
||
{"vs3:9333", 1.0, 500, blockvol.RoleReplica, now},
|
||
},
|
||
500,
|
||
)
|
||
|
||
var wg sync.WaitGroup
|
||
const rounds = 100
|
||
|
||
// Simulate heartbeats with varying health scores
|
||
// (as would happen during scrub with concurrent writes).
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for i := 0; i < rounds; i++ {
|
||
// Health stays 1.0 — no corruption detected (correct behavior
|
||
// when writes are excluded from scrub checks).
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-scrub.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||
WalHeadLsn: uint64(500 + i),
|
||
HealthScore: 1.0, // Clean — no false positives
|
||
},
|
||
}, "")
|
||
}
|
||
}()
|
||
|
||
// Concurrent replica heartbeats.
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for i := 0; i < rounds; i++ {
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-scrub.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
WalHeadLsn: uint64(500 + i),
|
||
HealthScore: 1.0,
|
||
},
|
||
}, "")
|
||
}
|
||
}()
|
||
|
||
wg.Wait()
|
||
|
||
entry, _ := ms.blockRegistry.Lookup("vol-scrub")
|
||
if entry.HealthScore < 1.0 {
|
||
t.Fatalf("health score should be 1.0 (no corruption), got %f", entry.HealthScore)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-10: ScrubDetectsCorruption_HealthDrops_PromotionAvoids
|
||
//
|
||
// One replica has corruption (low health), the other is clean.
|
||
// Promotion must prefer the healthy replica.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_ScrubDetectsCorruption_HealthDrops_PromotionAvoids(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
now := time.Now()
|
||
qaRegisterRF3WithState(t, ms, "vol-corrupt", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 0.3, 100, blockvol.RoleReplica, now}, // corrupted (low health)
|
||
{"vs3:9333", 1.0, 100, blockvol.RoleReplica, now}, // clean
|
||
},
|
||
100,
|
||
)
|
||
|
||
// Simulate heartbeat reporting low health on vs2 (scrub found errors).
|
||
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
|
||
{
|
||
Path: "/data/vol-corrupt.blk",
|
||
VolumeSize: 1 << 30,
|
||
Epoch: 1,
|
||
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
||
WalHeadLsn: 100,
|
||
HealthScore: 0.3,
|
||
ScrubErrors: 5,
|
||
},
|
||
}, "")
|
||
|
||
// Trigger failover — vs3 (healthy) should be promoted, not vs2.
|
||
newEpoch, err := ms.blockRegistry.PromoteBestReplica("vol-corrupt")
|
||
if err != nil {
|
||
t.Fatalf("promotion should succeed: %v", err)
|
||
}
|
||
if newEpoch != 2 {
|
||
t.Fatalf("epoch: got %d, want 2", newEpoch)
|
||
}
|
||
|
||
entry, _ := ms.blockRegistry.Lookup("vol-corrupt")
|
||
if entry.VolumeServer != "vs3:9333" {
|
||
t.Fatalf("vs3 (healthy) should be promoted, got %q", entry.VolumeServer)
|
||
}
|
||
// vs2 (corrupted) should remain as replica.
|
||
if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs2:9333" {
|
||
t.Fatalf("vs2 should remain as replica, got %+v", entry.Replicas)
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-11: ExpandRF3_PartialReplicaFailure
|
||
//
|
||
// Primary expand succeeds, one replica expand fails.
|
||
// Verify: operation succeeds (best-effort), registry updated,
|
||
// failed replica state is degraded.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_ExpandRF3_PartialReplicaFailure(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
ctx := context.Background()
|
||
|
||
// Create RF=3 volume first.
|
||
_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
|
||
Name: "vol-expand",
|
||
SizeBytes: 1 << 30,
|
||
ReplicaFactor: 3,
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("create: %v", err)
|
||
}
|
||
|
||
entry, _ := ms.blockRegistry.Lookup("vol-expand")
|
||
failServer := entry.Replicas[1].Server
|
||
|
||
// CP11A-2: coordinated expand — set up prepare/commit/cancel mocks.
|
||
ms.blockVSPrepareExpand = func(ctx context.Context, server string, name string, newSize, expandEpoch uint64) error {
|
||
return nil
|
||
}
|
||
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) {
|
||
if server == failServer {
|
||
return 0, fmt.Errorf("disk full on %s", server)
|
||
}
|
||
return 2 << 30, nil
|
||
}
|
||
ms.blockVSCancelExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) error {
|
||
return nil
|
||
}
|
||
|
||
// Under coordinated expand, partial replica commit failure marks the volume degraded.
|
||
_, err = ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
|
||
Name: "vol-expand",
|
||
NewSizeBytes: 2 << 30,
|
||
})
|
||
if err == nil {
|
||
t.Fatal("expand should fail when a required replica commit fails")
|
||
}
|
||
|
||
// Registry size should NOT be updated (primary committed but replica failed → degraded).
|
||
entry, _ = ms.blockRegistry.Lookup("vol-expand")
|
||
if entry.SizeBytes != 1<<30 {
|
||
t.Fatalf("registry size should be unchanged: got %d, want %d", entry.SizeBytes, uint64(1<<30))
|
||
}
|
||
if !entry.ExpandFailed {
|
||
t.Fatal("ExpandFailed should be true after partial commit failure")
|
||
}
|
||
if !entry.ExpandInProgress {
|
||
t.Fatal("ExpandInProgress should stay true to suppress heartbeat size updates")
|
||
}
|
||
|
||
// Cleanup: ClearExpandFailed allows future operations.
|
||
ms.blockRegistry.ClearExpandFailed("vol-expand")
|
||
|
||
// Now expand with all mocks succeeding should work.
|
||
ms.blockVSCommitExpand = func(ctx context.Context, server string, name string, expandEpoch uint64) (uint64, error) {
|
||
return 2 << 30, nil
|
||
}
|
||
resp, err := ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
|
||
Name: "vol-expand",
|
||
NewSizeBytes: 2 << 30,
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("retry expand after clear: %v", err)
|
||
}
|
||
if resp.CapacityBytes != 2<<30 {
|
||
t.Fatalf("capacity: got %d, want %d", resp.CapacityBytes, uint64(2<<30))
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// QA-12: ByServerIndexConsistency_AfterReplicaMove
|
||
//
|
||
// Moves replica between servers repeatedly. ListByServer and
|
||
// byServer counts must stay exact — no dangling references.
|
||
// ────────────────────────────────────────────────────────────
|
||
func TestQA_CP82_ByServerIndexConsistency_AfterReplicaMove(t *testing.T) {
|
||
ms := qaCP82Master(t)
|
||
now := time.Now()
|
||
qaRegisterRF3WithState(t, ms, "vol-move", "vs1:9333",
|
||
[]struct {
|
||
Server string
|
||
HealthScore float64
|
||
WALHeadLSN uint64
|
||
Role blockvol.Role
|
||
Heartbeat time.Time
|
||
}{
|
||
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, now},
|
||
},
|
||
100,
|
||
)
|
||
|
||
// Verify initial state: vs1 and vs2 both have vol-move.
|
||
assertServerHasVolume(t, ms, "vs1:9333", "vol-move")
|
||
assertServerHasVolume(t, ms, "vs2:9333", "vol-move")
|
||
|
||
// Move replica from vs2 to vs3 repeatedly.
|
||
servers := []string{"vs3:9333", "vs2:9333", "vs3:9333", "vs2:9333", "vs3:9333"}
|
||
for _, newServer := range servers {
|
||
// Remove old replica.
|
||
entry, _ := ms.blockRegistry.Lookup("vol-move")
|
||
oldServer := ""
|
||
if len(entry.Replicas) > 0 {
|
||
oldServer = entry.Replicas[0].Server
|
||
}
|
||
if oldServer != "" {
|
||
if err := ms.blockRegistry.RemoveReplica("vol-move", oldServer); err != nil {
|
||
t.Fatalf("remove replica from %s: %v", oldServer, err)
|
||
}
|
||
}
|
||
|
||
// Add new replica.
|
||
if err := ms.blockRegistry.AddReplica("vol-move", ReplicaInfo{
|
||
Server: newServer,
|
||
Path: "/data/vol-move.blk",
|
||
IQN: fmt.Sprintf("iqn.2024.test:vol-move-r"),
|
||
ISCSIAddr: newServer + ":3260",
|
||
HealthScore: 1.0,
|
||
}); err != nil {
|
||
t.Fatalf("add replica on %s: %v", newServer, err)
|
||
}
|
||
|
||
// Invariant: primary always indexed.
|
||
assertServerHasVolume(t, ms, "vs1:9333", "vol-move")
|
||
// New replica indexed.
|
||
assertServerHasVolume(t, ms, newServer, "vol-move")
|
||
// Old server should NOT have it (unless it's the primary).
|
||
if oldServer != "" && oldServer != "vs1:9333" && oldServer != newServer {
|
||
assertServerDoesNotHaveVolume(t, ms, oldServer, "vol-move")
|
||
}
|
||
}
|
||
|
||
// Final check: ListAll should have exactly 1 volume.
|
||
all := ms.blockRegistry.ListAll()
|
||
if len(all) != 1 {
|
||
t.Fatalf("expected 1 volume, got %d", len(all))
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// Helpers
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
func assertServerHasVolume(t *testing.T, ms *MasterServer, server, volName string) {
|
||
t.Helper()
|
||
entries := ms.blockRegistry.ListByServer(server)
|
||
for _, e := range entries {
|
||
if e.Name == volName {
|
||
return
|
||
}
|
||
}
|
||
t.Errorf("server %q should have volume %q in byServer index", server, volName)
|
||
}
|
||
|
||
func assertServerDoesNotHaveVolume(t *testing.T, ms *MasterServer, server, volName string) {
|
||
t.Helper()
|
||
entries := ms.blockRegistry.ListByServer(server)
|
||
for _, e := range entries {
|
||
if e.Name == volName {
|
||
t.Errorf("server %q should NOT have volume %q in byServer index", server, volName)
|
||
return
|
||
}
|
||
}
|
||
}
|