Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-20 13:30:46 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
fbcfe89e2461d42158cdc5e678d64e281ec7d2c8
seaweedfs/sw-block/runtime/volumev2/poc_test.go
T
pingqiuandClaude Opus 4.6 044a6d770b feat: Phase 19 — bounded working RF2 block path 
Live HTTP evidence transport, continuous Loop2 service, bounded auto
failover trigger, runtime-managed frontend export, bounded replica
repair, end-to-end RF2 handoff with continued I/O on new primary,
bounded operator HTTP surface, and CSI V2 runtime backend adapter.

11 new proof tests covering the full M6-M10 chain plus CSI create/
lookup/publish through the V2 runtime path.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-05 15:12:00 -07:00

3180 lines
94 KiB
Go
Raw Blame History

package volumev2
import (
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
"net/http"
"path/filepath"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2"
"github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi"
)
func TestPOC_MasterV2VolumeV2_RF1HeartbeatAssignmentFlow(t *testing.T) {
master := masterv2.New(masterv2.Config{})
node, err := New(Config{NodeID: "node-a"})
if err != nil {
t.Fatalf("new volumev2 node: %v", err)
}
defer node.Close()
session, err := NewInProcessSession(master)
if err != nil {
t.Fatalf("new session: %v", err)
}
orchestrator, err := NewOrchestrator(node, session)
if err != nil {
t.Fatalf("new orchestrator: %v", err)
}
path := filepath.Join(t.TempDir(), "rf1-poc.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "vol-a",
Path: path,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
assignments, err := session.Sync(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("preview sync: %v", err)
}
if len(assignments) != 1 {
t.Fatalf("preview assignments=%d, want 1", len(assignments))
}
if assignments[0].Epoch != 1 {
t.Fatalf("epoch=%d, want 1", assignments[0].Epoch)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync once 1: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync once 2: %v", err)
}
view, ok := master.Volume("vol-a")
if !ok {
t.Fatal("master view missing")
}
if view.ObservedRole != "primary" {
t.Fatalf("observed role=%q", view.ObservedRole)
}
if !view.RoleApplied {
t.Fatal("role_applied should be true")
}
if view.Mode != "allocated_only" {
t.Fatalf("mode=%q", view.Mode)
}
payload := bytes.Repeat([]byte{0x4A}, 4096)
if err := node.WriteLBA("vol-a", 0, payload); err != nil {
t.Fatalf("write: %v", err)
}
if err := node.SyncCache("vol-a"); err != nil {
t.Fatalf("sync cache: %v", err)
}
readBack, err := node.ReadLBA("vol-a", 0, uint32(len(payload)))
if err != nil {
t.Fatalf("read: %v", err)
}
if !bytes.Equal(readBack, payload) {
t.Fatal("payload mismatch")
}
}
func TestPOC_MasterV2VolumeV2_ReissuesAssignmentAfterDesiredPathChange(t *testing.T) {
master := masterv2.New(masterv2.Config{})
node, err := New(Config{NodeID: "node-a"})
if err != nil {
t.Fatalf("new volumev2 node: %v", err)
}
defer node.Close()
tempDir := t.TempDir()
path1 := filepath.Join(tempDir, "vol-a-1.blk")
path2 := filepath.Join(tempDir, "vol-a-2.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "vol-a",
Path: path1,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary path1: %v", err)
}
assignments, err := master.HandleHeartbeat(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("handle heartbeat 0: %v", err)
}
if err := node.ApplyAssignments(assignments); err != nil {
t.Fatalf("apply assignments path1: %v", err)
}
if _, err := master.HandleHeartbeat(mustHeartbeat(t, node)); err != nil {
t.Fatalf("handle heartbeat after path1: %v", err)
}
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "vol-a",
Path: path2,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary path2: %v", err)
}
assignments, err = master.HandleHeartbeat(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("handle heartbeat for path change: %v", err)
}
if len(assignments) != 1 {
t.Fatalf("assignments=%d, want 1", len(assignments))
}
if assignments[0].Epoch != 2 {
t.Fatalf("epoch=%d, want 2", assignments[0].Epoch)
}
if assignments[0].Path != path2 {
t.Fatalf("path=%q, want %q", assignments[0].Path, path2)
}
if err := node.ApplyAssignments(assignments); err != nil {
t.Fatalf("apply assignments path2: %v", err)
}
assignments, err = master.HandleHeartbeat(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("handle heartbeat after path2: %v", err)
}
if len(assignments) != 0 {
t.Fatalf("unexpected reissue count=%d", len(assignments))
}
view, ok := master.Volume("vol-a")
if !ok {
t.Fatal("master view missing")
}
if view.Path != path2 {
t.Fatalf("view path=%q, want %q", view.Path, path2)
}
if view.ObservedEpoch != 2 {
t.Fatalf("observed epoch=%d, want 2", view.ObservedEpoch)
}
}
func TestPOC_MasterV2VolumeV2_OrchestratorSyncsSingleNodeMVP(t *testing.T) {
master := masterv2.New(masterv2.Config{})
node, err := New(Config{NodeID: "node-a"})
if err != nil {
t.Fatalf("new volumev2 node: %v", err)
}
defer node.Close()
session, err := NewInProcessSession(master)
if err != nil {
t.Fatalf("new session: %v", err)
}
orchestrator, err := NewOrchestrator(node, session)
if err != nil {
t.Fatalf("new orchestrator: %v", err)
}
path := filepath.Join(t.TempDir(), "vol-orch.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "vol-orch",
Path: path,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync once 1: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync once 2: %v", err)
}
snap, err := node.Snapshot("vol-orch")
if err != nil {
t.Fatalf("snapshot: %v", err)
}
if !snap.HasProjection || !snap.Projection.Readiness.RoleApplied {
t.Fatal("expected applied projection after orchestrator sync")
}
}
func TestKernelControlPlaneClosure_ConvergesAndReissuesOnDesiredChange(t *testing.T) {
master := masterv2.New(masterv2.Config{})
node, err := New(Config{NodeID: "node-a"})
if err != nil {
t.Fatalf("new volumev2 node: %v", err)
}
defer node.Close()
session, err := NewInProcessSession(master)
if err != nil {
t.Fatalf("new session: %v", err)
}
orchestrator, err := NewOrchestrator(node, session)
if err != nil {
t.Fatalf("new orchestrator: %v", err)
}
tempDir := t.TempDir()
path1 := filepath.Join(tempDir, "control-a.blk")
path2 := filepath.Join(tempDir, "control-b.blk")
spec := masterv2.VolumeSpec{
Name: "control-vol",
Path: path1,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}
if err := master.DeclarePrimary(spec); err != nil {
t.Fatalf("declare primary path1: %v", err)
}
assignments, err := session.Sync(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("preview sync before convergence: %v", err)
}
if len(assignments) != 1 {
t.Fatalf("preview assignments=%d, want 1", len(assignments))
}
if assignments[0].Path != path1 || assignments[0].Epoch != 1 {
t.Fatalf("unexpected first assignment: %+v", assignments[0])
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 1: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 2: %v", err)
}
assignments, err = session.Sync(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("post-convergence sync: %v", err)
}
if len(assignments) != 0 {
t.Fatalf("unexpected post-convergence assignments=%d", len(assignments))
}
view, ok := master.Volume("control-vol")
if !ok {
t.Fatal("master view missing after convergence")
}
if view.DesiredEpoch != 1 || view.ObservedEpoch != 1 {
t.Fatalf("epochs after convergence desired=%d observed=%d", view.DesiredEpoch, view.ObservedEpoch)
}
if view.ObservedRole != "primary" || !view.RoleApplied {
t.Fatalf("view after convergence = %+v", view)
}
spec.Path = path2
if err := master.DeclarePrimary(spec); err != nil {
t.Fatalf("declare primary path2: %v", err)
}
assignments, err = session.Sync(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("sync after desired change: %v", err)
}
if len(assignments) != 1 {
t.Fatalf("assignments after desired change=%d, want 1", len(assignments))
}
if assignments[0].Path != path2 || assignments[0].Epoch != 2 {
t.Fatalf("unexpected reissued assignment: %+v", assignments[0])
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 3: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 4: %v", err)
}
assignments, err = session.Sync(mustHeartbeat(t, node))
if err != nil {
t.Fatalf("final sync after re-convergence: %v", err)
}
if len(assignments) != 0 {
t.Fatalf("unexpected assignments after re-convergence=%d", len(assignments))
}
view, ok = master.Volume("control-vol")
if !ok {
t.Fatal("master view missing after desired change")
}
if view.Path != path2 || view.DesiredEpoch != 2 || view.ObservedEpoch != 2 {
t.Fatalf("view after desired change = %+v", view)
}
if view.ObservedRole != "primary" || !view.RoleApplied {
t.Fatalf("final view not applied = %+v", view)
}
}
func TestLoop1_PromotionQueryUsesFreshEvidenceSeparateFromHeartbeat(t *testing.T) {
master := masterv2.New(masterv2.Config{})
node, err := New(Config{NodeID: "node-a"})
if err != nil {
t.Fatalf("new volumev2 node: %v", err)
}
defer node.Close()
session, err := NewInProcessSession(master)
if err != nil {
t.Fatalf("new session: %v", err)
}
orchestrator, err := NewOrchestrator(node, session)
if err != nil {
t.Fatalf("new orchestrator: %v", err)
}
path := filepath.Join(t.TempDir(), "query-vol.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "query-vol",
Path: path,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 1: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 2: %v", err)
}
hb := mustHeartbeat(t, node)
if len(hb.Volumes) != 1 {
t.Fatalf("heartbeat volumes=%d, want 1", len(hb.Volumes))
}
if hb.Volumes[0].Mode != "allocated_only" {
t.Fatalf("heartbeat mode=%q", hb.Volumes[0].Mode)
}
before, err := node.QueryPromotionEvidence(masterv2.PromotionQueryRequest{
VolumeName: "query-vol",
ExpectedEpoch: 1,
})
if err != nil {
t.Fatalf("query before write: %v", err)
}
if !before.Eligible {
t.Fatalf("before write should be eligible: %+v", before)
}
if hb.Volumes[0].CommittedLSN != before.CommittedLSN {
t.Fatalf("heartbeat committed_lsn=%d, query committed_lsn=%d", hb.Volumes[0].CommittedLSN, before.CommittedLSN)
}
payload := bytes.Repeat([]byte{0x7D}, 4096)
if err := node.WriteLBA("query-vol", 0, payload); err != nil {
t.Fatalf("write: %v", err)
}
if err := node.SyncCache("query-vol"); err != nil {
t.Fatalf("sync cache: %v", err)
}
after, err := node.QueryPromotionEvidence(masterv2.PromotionQueryRequest{
VolumeName: "query-vol",
ExpectedEpoch: 1,
})
if err != nil {
t.Fatalf("query after write: %v", err)
}
if after.CommittedLSN <= before.CommittedLSN {
t.Fatalf("committed_lsn before=%d after=%d", before.CommittedLSN, after.CommittedLSN)
}
if after.WALHeadLSN <= before.WALHeadLSN {
t.Fatalf("wal_head_lsn before=%d after=%d", before.WALHeadLSN, after.WALHeadLSN)
}
if after.CommittedLSN <= hb.Volumes[0].CommittedLSN {
t.Fatalf("fresh query committed_lsn=%d should advance beyond heartbeat cache=%d", after.CommittedLSN, hb.Volumes[0].CommittedLSN)
}
if _, err := master.HandleHeartbeat(mustHeartbeat(t, node)); err != nil {
t.Fatalf("refresh heartbeat after write: %v", err)
}
view, ok := master.Volume("query-vol")
if !ok {
t.Fatal("master view missing after refreshed heartbeat")
}
if view.CommittedLSN != after.CommittedLSN {
t.Fatalf("master cached committed_lsn=%d, want %d", view.CommittedLSN, after.CommittedLSN)
}
}
func TestLoop2_ReplicaSummaryPreservesBoundaryLayers(t *testing.T) {
master := masterv2.New(masterv2.Config{})
node, err := New(Config{NodeID: "node-a"})
if err != nil {
t.Fatalf("new volumev2 node: %v", err)
}
defer node.Close()
session, err := NewInProcessSession(master)
if err != nil {
t.Fatalf("new session: %v", err)
}
orchestrator, err := NewOrchestrator(node, session)
if err != nil {
t.Fatalf("new orchestrator: %v", err)
}
path := filepath.Join(t.TempDir(), "summary-vol.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "summary-vol",
Path: path,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 1: %v", err)
}
if err := orchestrator.SyncOnce(); err != nil {
t.Fatalf("sync 2: %v", err)
}
payload := bytes.Repeat([]byte{0x22}, 4096)
if err := node.WriteLBA("summary-vol", 0, payload); err != nil {
t.Fatalf("write: %v", err)
}
beforeSync, err := node.QueryReplicaSummary(protocolv2.ReplicaSummaryRequest{
VolumeName: "summary-vol",
ExpectedEpoch: 1,
})
if err != nil {
t.Fatalf("summary before sync: %v", err)
}
if !beforeSync.Eligible {
t.Fatalf("summary before sync should be eligible: %+v", beforeSync)
}
if beforeSync.CommittedLSN == 0 {
t.Fatalf("committed_lsn=%d, want > 0", beforeSync.CommittedLSN)
}
if beforeSync.DurableLSN != 0 {
t.Fatalf("durable_lsn=%d before sync, want 0", beforeSync.DurableLSN)
}
if err := node.SyncCache("summary-vol"); err != nil {
t.Fatalf("sync cache: %v", err)
}
afterSync, err := node.QueryReplicaSummary(protocolv2.ReplicaSummaryRequest{
VolumeName: "summary-vol",
ExpectedEpoch: 1,
})
if err != nil {
t.Fatalf("summary after sync: %v", err)
}
if afterSync.CommittedLSN < beforeSync.CommittedLSN {
t.Fatalf("committed_lsn before=%d after=%d", beforeSync.CommittedLSN, afterSync.CommittedLSN)
}
if afterSync.DurableLSN < afterSync.CommittedLSN {
t.Fatalf("durable_lsn=%d committed_lsn=%d", afterSync.DurableLSN, afterSync.CommittedLSN)
}
if afterSync.CheckpointLSN > afterSync.DurableLSN {
t.Fatalf("checkpoint_lsn=%d durable_lsn=%d", afterSync.CheckpointLSN, afterSync.DurableLSN)
}
if afterSync.RecoveryPhase != "idle" {
t.Fatalf("recovery_phase=%q", afterSync.RecoveryPhase)
}
}
func TestLoop2_ReplacementPrimaryReconstructsTakeoverTruth(t *testing.T) {
master := masterv2.New(masterv2.Config{})
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
tempDir := t.TempDir()
assignments := []masterv2.Assignment{
{
Name: "takeover-vol",
Path: filepath.Join(tempDir, "takeover-b.blk"),
NodeID: "node-b",
Epoch: 3,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
},
{
Name: "takeover-vol",
Path: filepath.Join(tempDir, "takeover-c.blk"),
NodeID: "node-c",
Epoch: 3,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
},
}
if err := nodeB.ApplyAssignments(assignments); err != nil {
t.Fatalf("apply node-b assignment: %v", err)
}
if err := nodeC.ApplyAssignments(assignments); err != nil {
t.Fatalf("apply node-c assignment: %v", err)
}
payload := bytes.Repeat([]byte{0x6E}, 4096)
if err := nodeB.WriteLBA("takeover-vol", 0, payload); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("takeover-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
candidate, err := master.SelectPromotionCandidate([]masterv2.PromotionQueryResponse{
mustPromotionEvidence(t, nodeB, "takeover-vol", 3),
mustPromotionEvidence(t, nodeC, "takeover-vol", 3),
})
if err != nil {
t.Fatalf("select promotion candidate: %v", err)
}
if candidate.NodeID != "node-b" {
t.Fatalf("candidate node=%q, want node-b", candidate.NodeID)
}
truth, err := nodeB.ReconstructTakeoverTruth("takeover-vol", 3, []ReplicaSummarySource{nodeC})
if err != nil {
t.Fatalf("reconstruct takeover truth: %v", err)
}
if truth.PrimaryNodeID != "node-b" {
t.Fatalf("primary_node=%q, want node-b", truth.PrimaryNodeID)
}
if truth.ReplicaCount != 2 {
t.Fatalf("replica_count=%d, want 2", truth.ReplicaCount)
}
if truth.CommittedLSN == 0 {
t.Fatalf("committed_lsn=%d, want > 0", truth.CommittedLSN)
}
if truth.DurableLSN < truth.CommittedLSN {
t.Fatalf("durable_lsn=%d committed_lsn=%d", truth.DurableLSN, truth.CommittedLSN)
}
if truth.Degraded {
t.Fatalf("unexpected degraded truth: %+v", truth)
}
}
func TestLoop2_PrepareAndGatePrimaryTakeover_AllowsHealthyCandidate(t *testing.T) {
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
tempDir := t.TempDir()
assignB := masterv2.Assignment{
Name: "takeover-allow-vol",
Path: filepath.Join(tempDir, "takeover-allow-b.blk"),
NodeID: "node-b",
Epoch: 5,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}
assignC := masterv2.Assignment{
Name: "takeover-allow-vol",
Path: filepath.Join(tempDir, "takeover-allow-c.blk"),
NodeID: "node-c",
Epoch: 5,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{assignC}); err != nil {
t.Fatalf("apply peer assignment: %v", err)
}
payload := bytes.Repeat([]byte{0x2C}, 4096)
if err := nodeB.ApplyAssignments([]masterv2.Assignment{assignB}); err != nil {
t.Fatalf("seed candidate assignment: %v", err)
}
if err := nodeB.WriteLBA("takeover-allow-vol", 0, payload); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("takeover-allow-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
truth, err := nodeB.PreparePrimaryTakeover(PrimaryTakeoverPlan{
Assignment: assignB,
Peers: []ReplicaSummarySource{nodeC},
})
if err != nil {
t.Fatalf("prepare primary takeover: %v", err)
}
if err := nodeB.GatePrimaryActivation(assignB.Name, truth); err != nil {
t.Fatalf("gate primary activation: %v", err)
}
if truth.PrimaryNodeID != "node-b" {
t.Fatalf("primary_node=%q, want node-b", truth.PrimaryNodeID)
}
if truth.ReplicaCount != 2 {
t.Fatalf("replica_count=%d, want 2", truth.ReplicaCount)
}
}
func TestLoop2_ApplyPrimaryTakeover_GatesDegradedTruth(t *testing.T) {
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
tempDir := t.TempDir()
assignB := masterv2.Assignment{
Name: "takeover-gated-vol",
Path: filepath.Join(tempDir, "takeover-gated-b.blk"),
NodeID: "node-b",
Epoch: 6,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}
truth, err := nodeB.PreparePrimaryTakeover(PrimaryTakeoverPlan{
Assignment: assignB,
Peers: []ReplicaSummarySource{
staticReplicaSummarySource{resp: protocolv2.ReplicaSummaryResponse{
VolumeName: "takeover-gated-vol",
NodeID: "node-c",
Epoch: 5,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
}},
},
})
if err != nil {
t.Fatalf("prepare primary takeover: %v", err)
}
if err := nodeB.GatePrimaryActivation(assignB.Name, truth); err == nil {
t.Fatal("expected takeover gate error")
}
}
func TestFailoverFlow_AuthorizesAndActivatesHealthyCandidate(t *testing.T) {
master := masterv2.New(masterv2.Config{})
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "flow-b.blk")
pathC := filepath.Join(tempDir, "flow-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "flow-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "flow-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "flow-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
payload := bytes.Repeat([]byte{0x5A}, 4096)
if err := nodeB.WriteLBA("flow-vol", 0, payload); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("flow-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
result, err := ExecuteFailoverFlow(master, "flow-vol", 2, []FailoverTarget{
mustInProcessFailoverTarget(t, nodeB),
mustInProcessFailoverTarget(t, nodeC),
})
if err != nil {
t.Fatalf("execute failover flow: %v", err)
}
if result.Candidate.NodeID != "node-b" {
t.Fatalf("candidate node=%q, want node-b", result.Candidate.NodeID)
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID)
}
if result.Assignment.Epoch != 2 {
t.Fatalf("assignment epoch=%d, want 2", result.Assignment.Epoch)
}
if result.Truth.PrimaryNodeID != "node-b" {
t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID)
}
if result.Truth.Degraded {
t.Fatalf("unexpected degraded truth: %+v", result.Truth)
}
view, ok := master.Volume("flow-vol")
if !ok {
t.Fatal("master view missing")
}
if view.PrimaryNodeID != "node-b" {
t.Fatalf("view primary=%q, want node-b", view.PrimaryNodeID)
}
if view.DesiredEpoch != 2 {
t.Fatalf("view desired epoch=%d, want 2", view.DesiredEpoch)
}
}
func TestFailoverFlow_StopsAtActivationGate(t *testing.T) {
master := masterv2.New(masterv2.Config{})
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "flow-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "flow-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "flow-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
result, err := ExecuteFailoverFlow(master, "flow-gated-vol", 2, []FailoverTarget{
mustInProcessFailoverTarget(t, nodeB),
staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "flow-gated-vol",
NodeID: "node-c",
Epoch: 1,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "flow-gated-vol",
NodeID: "node-c",
Epoch: 1,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
),
})
if err == nil {
t.Fatal("expected failover gate error")
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID)
}
if !result.Truth.Degraded {
t.Fatalf("expected degraded truth: %+v", result.Truth)
}
if !result.Truth.NeedsRebuild {
t.Fatalf("expected needs_rebuild truth: %+v", result.Truth)
}
}
func TestFailoverSession_StepwiseStagesExposeIntermediateState(t *testing.T) {
master := masterv2.New(masterv2.Config{})
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "session-b.blk")
pathC := filepath.Join(tempDir, "session-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "session-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "session-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "session-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := nodeB.WriteLBA("session-vol", 0, bytes.Repeat([]byte{0x41}, 4096)); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("session-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
session, err := NewFailoverSession(master, "session-vol", 2, []FailoverTarget{
mustInProcessFailoverTarget(t, nodeB),
mustInProcessFailoverTarget(t, nodeC),
})
if err != nil {
t.Fatalf("new failover session: %v", err)
}
if session.Stage() != FailoverStageNew {
t.Fatalf("initial stage=%q, want %q", session.Stage(), FailoverStageNew)
}
responses, err := session.CollectPromotionEvidence()
if err != nil {
t.Fatalf("collect promotion evidence: %v", err)
}
if len(responses) != 2 {
t.Fatalf("responses=%d, want 2", len(responses))
}
if session.Stage() != FailoverStageEvidenceCollected {
t.Fatalf("stage after collect=%q, want %q", session.Stage(), FailoverStageEvidenceCollected)
}
if session.Snapshot().ResponseCount != 2 {
t.Fatalf("snapshot response_count=%d, want 2", session.Snapshot().ResponseCount)
}
assign, err := session.Authorize()
if err != nil {
t.Fatalf("authorize: %v", err)
}
if assign.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", assign.NodeID)
}
if session.Stage() != FailoverStageAuthorized {
t.Fatalf("stage after authorize=%q, want %q", session.Stage(), FailoverStageAuthorized)
}
truth, err := session.PrepareTakeover()
if err != nil {
t.Fatalf("prepare takeover: %v", err)
}
if truth.PrimaryNodeID != "node-b" {
t.Fatalf("truth primary=%q, want node-b", truth.PrimaryNodeID)
}
if session.Stage() != FailoverStagePrepared {
t.Fatalf("stage after prepare=%q, want %q", session.Stage(), FailoverStagePrepared)
}
if err := session.Activate(); err != nil {
t.Fatalf("activate: %v", err)
}
if session.Stage() != FailoverStageActivated {
t.Fatalf("stage after activate=%q, want %q", session.Stage(), FailoverStageActivated)
}
result := session.Result()
if result.Candidate.NodeID != "node-b" {
t.Fatalf("result candidate=%q, want node-b", result.Candidate.NodeID)
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("result assignment=%q, want node-b", result.Assignment.NodeID)
}
if result.Truth.PrimaryNodeID != "node-b" {
t.Fatalf("result truth primary=%q, want node-b", result.Truth.PrimaryNodeID)
}
}
func TestFailoverSession_SnapshotCapturesFailureState(t *testing.T) {
master := masterv2.New(masterv2.Config{})
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "snapshot-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "snapshot-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "snapshot-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
session, err := NewFailoverSession(master, "snapshot-gated-vol", 2, []FailoverTarget{
mustInProcessFailoverTarget(t, nodeB),
staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "snapshot-gated-vol",
NodeID: "node-c",
Epoch: 1,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "snapshot-gated-vol",
NodeID: "node-c",
Epoch: 1,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
),
})
if err != nil {
t.Fatalf("new failover session: %v", err)
}
if _, err := session.Run(); err == nil {
t.Fatal("expected failover session error")
}
if session.Stage() != FailoverStageFailed {
t.Fatalf("stage after failure=%q, want %q", session.Stage(), FailoverStageFailed)
}
if session.LastError() == nil {
t.Fatal("expected last error")
}
snap := session.Snapshot()
if snap.LastError == "" {
t.Fatal("expected snapshot last error")
}
if snap.SelectedNodeID != "node-b" {
t.Fatalf("snapshot selected node=%q, want node-b", snap.SelectedNodeID)
}
if !snap.Result.Truth.NeedsRebuild {
t.Fatalf("expected needs_rebuild truth in snapshot: %+v", snap.Result.Truth)
}
}
func TestInProcessFailoverDriver_ExecuteHealthyFailover(t *testing.T) {
master := masterv2.New(masterv2.Config{})
driver, err := NewInProcessFailoverDriver(master)
if err != nil {
t.Fatalf("new failover driver: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := driver.RegisterTarget(mustInProcessFailoverTarget(t, nodeB)); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := driver.RegisterTarget(mustInProcessFailoverTarget(t, nodeC)); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "driver-b.blk")
pathC := filepath.Join(tempDir, "driver-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "driver-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "driver-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "driver-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := nodeB.WriteLBA("driver-vol", 0, bytes.Repeat([]byte{0x33}, 4096)); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("driver-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
result, err := driver.Execute("driver-vol", 2)
if err != nil {
t.Fatalf("driver execute: %v", err)
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID)
}
if result.Truth.PrimaryNodeID != "node-b" {
t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID)
}
}
func TestInProcessFailoverDriver_ExecuteStopsOnGate(t *testing.T) {
master := masterv2.New(masterv2.Config{})
driver, err := NewInProcessFailoverDriver(master)
if err != nil {
t.Fatalf("new failover driver: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := driver.RegisterTarget(mustInProcessFailoverTarget(t, nodeB)); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := driver.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "driver-gated-vol",
NodeID: "node-c",
Epoch: 1,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "driver-gated-vol",
NodeID: "node-c",
Epoch: 1,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
)); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "driver-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "driver-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "driver-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
_, err = driver.Execute("driver-gated-vol", 2)
if err == nil {
t.Fatal("expected gated driver execute error")
}
}
func TestInProcessRuntimeManager_ExecuteHealthyFailoverAndPersistSnapshot(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "manager-b.blk")
pathC := filepath.Join(tempDir, "manager-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "manager-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "manager-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "manager-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := nodeB.WriteLBA("manager-vol", 0, bytes.Repeat([]byte{0x52}, 4096)); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("manager-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
result, err := manager.ExecuteFailover("manager-vol", 2)
if err != nil {
t.Fatalf("manager execute failover: %v", err)
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID)
}
lastSnap, ok := manager.LastFailoverSnapshot()
if !ok {
t.Fatal("expected last failover snapshot")
}
if lastSnap.Stage != FailoverStageActivated {
t.Fatalf("snapshot stage=%q, want %q", lastSnap.Stage, FailoverStageActivated)
}
if lastSnap.SelectedNodeID != "node-b" {
t.Fatalf("snapshot selected node=%q, want node-b", lastSnap.SelectedNodeID)
}
perVolSnap, ok := manager.FailoverSnapshot("manager-vol")
if !ok {
t.Fatal("expected per-volume failover snapshot")
}
if perVolSnap.Result.Truth.PrimaryNodeID != "node-b" {
t.Fatalf("per-volume truth primary=%q, want node-b", perVolSnap.Result.Truth.PrimaryNodeID)
}
lastResult, ok := manager.LastFailoverResult()
if !ok {
t.Fatal("expected last failover result")
}
if lastResult.Assignment.NodeID != "node-b" {
t.Fatalf("last result assignment=%q, want node-b", lastResult.Assignment.NodeID)
}
}
func TestInProcessRuntimeManager_ExecuteStopsOnGateAndPersistsFailureSnapshot(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "manager-gated-vol",
NodeID: "node-c",
Epoch: 1,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "manager-gated-vol",
NodeID: "node-c",
Epoch: 1,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
)); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "manager-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "manager-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "manager-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
_, err = manager.ExecuteFailover("manager-gated-vol", 2)
if err == nil {
t.Fatal("expected manager failover gate error")
}
lastSnap, ok := manager.LastFailoverSnapshot()
if !ok {
t.Fatal("expected last failover snapshot")
}
if lastSnap.Stage != FailoverStageFailed {
t.Fatalf("snapshot stage=%q, want %q", lastSnap.Stage, FailoverStageFailed)
}
if lastSnap.LastError == "" {
t.Fatal("expected snapshot last error")
}
if !lastSnap.Result.Truth.NeedsRebuild {
t.Fatalf("expected needs_rebuild truth in snapshot: %+v", lastSnap.Result.Truth)
}
}
func TestTransportEvidenceAdapter_HealthyFailoverFlow(t *testing.T) {
master := masterv2.New(masterv2.Config{})
transport := NewInMemoryFailoverEvidenceTransport()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := transport.RegisterHandler("node-b", nodeB); err != nil {
t.Fatalf("register node-b handler: %v", err)
}
if err := transport.RegisterHandler("node-c", nodeC); err != nil {
t.Fatalf("register node-c handler: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "transport-b.blk")
pathC := filepath.Join(tempDir, "transport-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "transport-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "transport-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "transport-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := nodeB.WriteLBA("transport-vol", 0, bytes.Repeat([]byte{0x61}, 4096)); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("transport-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
result, err := ExecuteFailoverFlow(master, "transport-vol", 2, []FailoverTarget{
mustHybridFailoverTarget(t, nodeB, transport),
mustHybridFailoverTarget(t, nodeC, transport),
})
if err != nil {
t.Fatalf("transport-backed failover flow: %v", err)
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID)
}
if result.Truth.PrimaryNodeID != "node-b" {
t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID)
}
}
func TestTransportEvidenceAdapter_GatedFailoverFlow(t *testing.T) {
master := masterv2.New(masterv2.Config{})
transport := NewInMemoryFailoverEvidenceTransport()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := transport.RegisterHandler("node-b", nodeB); err != nil {
t.Fatalf("register node-b handler: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "transport-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "transport-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "transport-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
_, err = ExecuteFailoverFlow(master, "transport-gated-vol", 2, []FailoverTarget{
mustHybridFailoverTarget(t, nodeB, transport),
staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "transport-gated-vol",
NodeID: "node-c",
Epoch: 1,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "transport-gated-vol",
NodeID: "node-c",
Epoch: 1,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
),
})
if err == nil {
t.Fatal("expected gated transport-backed failover error")
}
}
func TestHTTPTransportEvidenceAdapter_HealthyFailoverFlow(t *testing.T) {
master := masterv2.New(masterv2.Config{})
transport := NewHTTPFailoverEvidenceTransport()
defer transport.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := transport.RegisterHandler("node-b", nodeB); err != nil {
t.Fatalf("register node-b handler: %v", err)
}
if err := transport.RegisterHandler("node-c", nodeC); err != nil {
t.Fatalf("register node-c handler: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "http-transport-b.blk")
pathC := filepath.Join(tempDir, "http-transport-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "http-transport-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "http-transport-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "http-transport-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := nodeB.WriteLBA("http-transport-vol", 0, bytes.Repeat([]byte{0x71}, 4096)); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("http-transport-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
result, err := ExecuteFailoverFlow(master, "http-transport-vol", 2, []FailoverTarget{
mustHybridFailoverTarget(t, nodeB, transport),
mustHybridFailoverTarget(t, nodeC, transport),
})
if err != nil {
t.Fatalf("http transport failover flow: %v", err)
}
if result.Assignment.NodeID != "node-b" {
t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID)
}
if result.Truth.PrimaryNodeID != "node-b" {
t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID)
}
}
func TestHTTPTransportEvidenceAdapter_GatedFailoverFlow(t *testing.T) {
master := masterv2.New(masterv2.Config{})
transport := NewHTTPFailoverEvidenceTransport()
defer transport.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := transport.RegisterHandler("node-b", nodeB); err != nil {
t.Fatalf("register node-b handler: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "http-transport-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "http-transport-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "http-transport-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
_, err = ExecuteFailoverFlow(master, "http-transport-gated-vol", 2, []FailoverTarget{
mustHybridFailoverTarget(t, nodeB, transport),
staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "http-transport-gated-vol",
NodeID: "node-c",
Epoch: 1,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "http-transport-gated-vol",
NodeID: "node-c",
Epoch: 1,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 3,
DurableLSN: 2,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
),
})
if err == nil {
t.Fatal("expected gated http transport failover error")
}
}
func TestInProcessRuntimeManager_Loop2Service_RefreshesRF2Surface(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport())
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "loop2-service-b.blk")
pathC := filepath.Join(tempDir, "loop2-service-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "loop2-service-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "loop2-service-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "loop2-service-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
handle, err := manager.StartLoop2Service(Loop2ServiceConfig{
VolumeName: "loop2-service-vol",
PrimaryNodeID: "node-b",
ExpectedEpoch: 2,
NodeIDs: []string{"node-b", "node-c"},
Interval: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("start loop2 service: %v", err)
}
defer func() {
handle.Stop()
handle.Wait()
}()
waitForCondition(t, time.Second, func() bool {
surface, ok := manager.RF2VolumeSurface("loop2-service-vol")
return ok && surface.HasLoop2 && surface.Mode == RF2SurfaceModeHealthy
})
}
func TestInProcessRuntimeManager_AutoFailoverService_TriggersOnPrimaryLoss(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport())
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "auto-failover-b.blk")
pathC := filepath.Join(tempDir, "auto-failover-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "auto-failover-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "auto-failover-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "auto-failover-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{
VolumeName: "auto-failover-vol",
PrimaryNodeID: "node-b",
ExpectedEpoch: 2,
NodeIDs: []string{"node-b", "node-c"},
Interval: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("start auto failover service: %v", err)
}
defer func() {
handle.Stop()
handle.Wait()
}()
manager.DisconnectEvidenceNode("node-b")
waitForCondition(t, time.Second, func() bool {
result, ok := manager.FailoverResult("auto-failover-vol")
return ok && result.Assignment.NodeID == "node-c"
})
if handle.PrimaryNodeID() != "node-c" {
t.Fatalf("tracked primary=%q, want node-c", handle.PrimaryNodeID())
}
}
func TestInProcessRuntimeManager_AutoFailoverService_DoesNotTriggerOnCatchingUpReplica(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport())
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "auto-suppress-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "auto-suppress-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "auto-suppress-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "auto-suppress-vol",
NodeID: "node-c",
Epoch: 2,
CommittedLSN: 1,
WALHeadLSN: 1,
ReceiverReady: true,
Eligible: true,
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "auto-suppress-vol",
NodeID: "node-c",
Epoch: 2,
Role: "replica",
CommittedLSN: 1,
DurableLSN: 1,
CheckpointLSN: 1,
TargetLSN: 2,
AchievedLSN: 1,
RecoveryPhase: "catching_up",
Eligible: true,
LastBarrierOK: true,
},
)); err != nil {
t.Fatalf("register node-c target: %v", err)
}
handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{
VolumeName: "auto-suppress-vol",
PrimaryNodeID: "node-b",
ExpectedEpoch: 2,
NodeIDs: []string{"node-b", "node-c"},
Interval: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("start auto failover service: %v", err)
}
defer func() {
handle.Stop()
handle.Wait()
}()
time.Sleep(120 * time.Millisecond)
if _, ok := manager.FailoverResult("auto-suppress-vol"); ok {
t.Fatal("unexpected auto failover result on catching_up replica")
}
surface, ok := manager.RF2VolumeSurface("auto-suppress-vol")
if !ok {
t.Fatal("expected rf2 volume surface")
}
if surface.Mode != RF2SurfaceModeCatchingUp {
t.Fatalf("surface mode=%q, want %q", surface.Mode, RF2SurfaceModeCatchingUp)
}
}
func TestInProcessRuntimeManager_ExportVolumeISCSI_BindsFrontendToRuntimeNode(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
pathB := filepath.Join(t.TempDir(), "frontend-runtime-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "frontend-runtime-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "frontend-runtime-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
export, err := manager.ExportVolumeISCSI("frontend-runtime-vol", "node-b", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.frontend-runtime-vol")
if err != nil {
t.Fatalf("export volume iscsi: %v", err)
}
snapshot, ok := manager.ISCSIExport("frontend-runtime-vol")
if !ok {
t.Fatal("expected managed iscsi export snapshot")
}
if snapshot.NodeID != "node-b" {
t.Fatalf("snapshot node=%q, want node-b", snapshot.NodeID)
}
conn := mustLoginISCSI(t, export.Address, export.IQN)
defer conn.Close()
writeData := bytes.Repeat([]byte{0x58}, 4096)
var writeCDB [16]byte
writeCDB[0] = iscsi.ScsiWrite10
binary.BigEndian.PutUint32(writeCDB[2:6], 0)
binary.BigEndian.PutUint16(writeCDB[7:9], 1)
resp := sendSCSICmd(t, conn, writeCDB, 2, false, true, writeData, uint32(len(writeData)))
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("iscsi write failed: status=%d", resp.SCSIStatus())
}
var syncCDB [16]byte
syncCDB[0] = iscsi.ScsiSyncCache10
resp = sendSCSICmd(t, conn, syncCDB, 3, false, false, nil, 0)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("iscsi sync cache failed: status=%d", resp.SCSIStatus())
}
readBack, err := nodeB.ReadLBA("frontend-runtime-vol", 0, uint32(len(writeData)))
if err != nil {
t.Fatalf("backend read: %v", err)
}
if !bytes.Equal(readBack, writeData) {
t.Fatal("backend readback mismatch")
}
}
func TestInProcessRuntimeManager_RepairReplicaFromPrimary_ReturnsLoop2ToHealthy(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport())
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "repair-b.blk")
pathC := filepath.Join(tempDir, "repair-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "repair-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
assignments := []masterv2.Assignment{
{
Name: "repair-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
},
{
Name: "repair-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
},
}
if err := nodeB.ApplyAssignments(assignments[:1]); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments(assignments[1:]); err != nil {
t.Fatalf("seed node-c: %v", err)
}
payload := bytes.Repeat([]byte{0x62}, 4096)
if err := nodeB.WriteLBA("repair-vol", 0, payload); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("repair-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
before, err := manager.ObserveLoop2("repair-vol", "node-b", 2, "node-b", "node-c")
if err != nil {
t.Fatalf("observe before repair: %v", err)
}
if before.Mode != Loop2RuntimeModeCatchingUp {
t.Fatalf("before repair mode=%q, want %q", before.Mode, Loop2RuntimeModeCatchingUp)
}
result, err := manager.RepairReplicaFromPrimary("repair-vol", "node-b", "node-c", 2, uint32(len(payload)))
if err != nil {
t.Fatalf("repair replica from primary: %v", err)
}
if result.Loop2Before.Mode != Loop2RuntimeModeCatchingUp {
t.Fatalf("repair before mode=%q, want %q", result.Loop2Before.Mode, Loop2RuntimeModeCatchingUp)
}
if result.Loop2After.Mode != Loop2RuntimeModeKeepUp {
t.Fatalf("repair after mode=%q, want %q", result.Loop2After.Mode, Loop2RuntimeModeKeepUp)
}
if !result.DataMatch {
t.Fatalf("repair data match=false: %+v", result)
}
}
func TestInProcessRuntimeManager_EndToEndRF2Handoff_ContinuesIOOnNewPrimary(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport())
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "e2e-b.blk")
pathC := filepath.Join(tempDir, "e2e-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "e2e-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "e2e-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "e2e-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
export, err := manager.ExportVolumeISCSI("e2e-vol", "node-b", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.e2e-vol.primary")
if err != nil {
t.Fatalf("export primary iscsi: %v", err)
}
conn := mustLoginISCSI(t, export.Address, export.IQN)
defer conn.Close()
firstPayload := bytes.Repeat([]byte{0x63}, 4096)
var writeCDB [16]byte
writeCDB[0] = iscsi.ScsiWrite10
binary.BigEndian.PutUint32(writeCDB[2:6], 0)
binary.BigEndian.PutUint16(writeCDB[7:9], 1)
resp := sendSCSICmd(t, conn, writeCDB, 2, false, true, firstPayload, uint32(len(firstPayload)))
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("initial iscsi write failed: status=%d", resp.SCSIStatus())
}
var syncCDB [16]byte
syncCDB[0] = iscsi.ScsiSyncCache10
resp = sendSCSICmd(t, conn, syncCDB, 3, false, false, nil, 0)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("initial iscsi sync failed: status=%d", resp.SCSIStatus())
}
if _, err := manager.RepairReplicaFromPrimary("e2e-vol", "node-b", "node-c", 2, uint32(len(firstPayload))); err != nil {
t.Fatalf("repair before failover: %v", err)
}
handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{
VolumeName: "e2e-vol",
PrimaryNodeID: "node-b",
ExpectedEpoch: 2,
NodeIDs: []string{"node-b", "node-c"},
Interval: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("start auto failover service: %v", err)
}
defer func() {
handle.Stop()
handle.Wait()
}()
manager.DisconnectEvidenceNode("node-b")
waitForCondition(t, time.Second, func() bool {
result, ok := manager.FailoverResult("e2e-vol")
return ok && result.Assignment.NodeID == "node-c"
})
exportAfter, err := manager.ExportCurrentPrimaryISCSI("e2e-vol", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.e2e-vol.failover")
if err != nil {
t.Fatalf("export new primary iscsi: %v", err)
}
connAfter := mustLoginISCSI(t, exportAfter.Address, exportAfter.IQN)
defer connAfter.Close()
var readCDB [16]byte
readCDB[0] = iscsi.ScsiRead10
binary.BigEndian.PutUint32(readCDB[2:6], 0)
binary.BigEndian.PutUint16(readCDB[7:9], 1)
resp = sendSCSICmd(t, connAfter, readCDB, 4, true, false, nil, uint32(len(firstPayload)))
if resp.Opcode() != iscsi.OpSCSIDataIn {
t.Fatalf("expected Data-In after failover, got %s", iscsi.OpcodeName(resp.Opcode()))
}
if !bytes.Equal(resp.DataSegment, firstPayload) {
t.Fatal("post-failover readback mismatch")
}
secondPayload := bytes.Repeat([]byte{0x64}, 4096)
resp = sendSCSICmd(t, connAfter, writeCDB, 5, false, true, secondPayload, uint32(len(secondPayload)))
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("post-failover write failed: status=%d", resp.SCSIStatus())
}
resp = sendSCSICmd(t, connAfter, syncCDB, 6, false, false, nil, 0)
if resp.SCSIStatus() != iscsi.SCSIStatusGood {
t.Fatalf("post-failover sync failed: status=%d", resp.SCSIStatus())
}
finalRead, err := nodeC.ReadLBA("e2e-vol", 0, uint32(len(secondPayload)))
if err != nil {
t.Fatalf("node-c backend read: %v", err)
}
if !bytes.Equal(finalRead, secondPayload) {
t.Fatal("post-failover backend readback mismatch")
}
}
func TestInProcessRuntimeManager_EndToEndRF2Handoff_GatedReplicaStopsFailClosed(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport())
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "e2e-gated-b.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "e2e-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "e2e-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "e2e-gated-vol",
NodeID: "node-c",
Epoch: 2,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "e2e-gated-vol",
NodeID: "node-c",
Epoch: 2,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 1,
DurableLSN: 1,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
)); err != nil {
t.Fatalf("register node-c target: %v", err)
}
handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{
VolumeName: "e2e-gated-vol",
PrimaryNodeID: "node-b",
ExpectedEpoch: 2,
NodeIDs: []string{"node-b", "node-c"},
Interval: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("start auto failover service: %v", err)
}
defer func() {
handle.Stop()
handle.Wait()
}()
manager.DisconnectEvidenceNode("node-b")
waitForCondition(t, time.Second, func() bool {
snap, ok := manager.FailoverSnapshot("e2e-gated-vol")
return ok && snap.Stage == FailoverStageFailed
})
result, ok := manager.FailoverResult("e2e-gated-vol")
if !ok {
t.Fatal("expected recorded gated failover result slot")
}
if result.Assignment.NodeID != "" {
t.Fatalf("unexpected successful assignment on gated handoff: %+v", result)
}
}
func TestInProcessRuntimeManager_OperatorSurface_ExposesRuntimeOwnedViews(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
defer manager.Close()
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "operator-b.blk")
pathC := filepath.Join(tempDir, "operator-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "operator-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "operator-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "operator-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if _, err := manager.ObserveLoop2("operator-vol", "node-b", 2, "node-b", "node-c"); err != nil {
t.Fatalf("observe loop2: %v", err)
}
if _, err := manager.ExportVolumeISCSI("operator-vol", "node-b", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.operator-vol"); err != nil {
t.Fatalf("export iscsi: %v", err)
}
handle, err := manager.StartOperatorSurface("127.0.0.1:0")
if err != nil {
t.Fatalf("start operator surface: %v", err)
}
defer handle.Close()
resp, err := http.Get("http://" + handle.Address() + "/v1/volumes/operator-vol/rf2")
if err != nil {
t.Fatalf("get rf2 surface: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("rf2 status=%d, want 200", resp.StatusCode)
}
var rf2Surface RF2VolumeSurface
if err := json.NewDecoder(resp.Body).Decode(&rf2Surface); err != nil {
t.Fatalf("decode rf2 surface: %v", err)
}
if rf2Surface.Mode != RF2SurfaceModeHealthy {
t.Fatalf("rf2 surface mode=%q, want %q", rf2Surface.Mode, RF2SurfaceModeHealthy)
}
resp, err = http.Get("http://" + handle.Address() + "/v1/volumes/operator-vol/frontend")
if err != nil {
t.Fatalf("get frontend surface: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("frontend status=%d, want 200", resp.StatusCode)
}
var frontend ManagedISCSIExportSnapshot
if err := json.NewDecoder(resp.Body).Decode(&frontend); err != nil {
t.Fatalf("decode frontend surface: %v", err)
}
if frontend.NodeID != "node-b" {
t.Fatalf("frontend node=%q, want node-b", frontend.NodeID)
}
}
func TestLoop2RuntimeSession_KeepUpOnHealthyReplicaSet(t *testing.T) {
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "loop2-keepup-b.blk")
assignB := masterv2.Assignment{
Name: "loop2-keepup-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 3,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{assignB}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
payload := bytes.Repeat([]byte{0x71}, 4096)
if err := nodeB.WriteLBA("loop2-keepup-vol", 0, payload); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("loop2-keepup-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
primary := mustReplicaSummary(t, nodeB, "loop2-keepup-vol", 3)
session, err := NewLoop2RuntimeSession("loop2-keepup-vol", "node-b", 3, []FailoverTarget{
mustInProcessFailoverTarget(t, nodeB),
staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{VolumeName: "loop2-keepup-vol", NodeID: "node-c"},
protocolv2.ReplicaSummaryResponse{
VolumeName: "loop2-keepup-vol",
NodeID: "node-c",
Epoch: 3,
Role: "replica",
Mode: "replica_ready",
RoleApplied: true,
ReceiverReady: true,
CommittedLSN: primary.CommittedLSN,
DurableLSN: primary.DurableLSN,
CheckpointLSN: primary.CheckpointLSN,
LastBarrierOK: true,
Eligible: true,
},
),
})
if err != nil {
t.Fatalf("new loop2 runtime session: %v", err)
}
snap, err := session.ObserveOnce()
if err != nil {
t.Fatalf("observe loop2 keepup: %v", err)
}
if snap.Mode != Loop2RuntimeModeKeepUp {
t.Fatalf("loop2 mode=%q, want %q", snap.Mode, Loop2RuntimeModeKeepUp)
}
if snap.HealthyReplicaCount != 2 {
t.Fatalf("healthy replicas=%d, want 2", snap.HealthyReplicaCount)
}
}
func TestInProcessRuntimeManager_ObserveLoop2_CatchingUp(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "loop2-catchup-b.blk")
assignB := masterv2.Assignment{
Name: "loop2-catchup-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 4,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{assignB}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
payload := bytes.Repeat([]byte{0x72}, 4096)
if err := nodeB.WriteLBA("loop2-catchup-vol", 0, payload); err != nil {
t.Fatalf("write node-b: %v", err)
}
if err := nodeB.SyncCache("loop2-catchup-vol"); err != nil {
t.Fatalf("sync node-b: %v", err)
}
primary := mustReplicaSummary(t, nodeB, "loop2-catchup-vol", 4)
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{VolumeName: "loop2-catchup-vol", NodeID: "node-c"},
protocolv2.ReplicaSummaryResponse{
VolumeName: "loop2-catchup-vol",
NodeID: "node-c",
Epoch: 4,
Role: "replica",
Mode: "replica_ready",
RoleApplied: true,
ReceiverReady: true,
CommittedLSN: primary.CommittedLSN - 1,
DurableLSN: primary.CommittedLSN - 1,
CheckpointLSN: primary.CheckpointLSN,
TargetLSN: primary.CommittedLSN,
AchievedLSN: primary.CommittedLSN - 1,
RecoveryPhase: "catching_up",
LastBarrierOK: true,
Eligible: true,
},
)); err != nil {
t.Fatalf("register node-c target: %v", err)
}
snap, err := manager.ObserveLoop2("loop2-catchup-vol", "node-b", 4)
if err != nil {
t.Fatalf("observe loop2 catchup: %v", err)
}
if snap.Mode != Loop2RuntimeModeCatchingUp {
t.Fatalf("loop2 mode=%q, want %q", snap.Mode, Loop2RuntimeModeCatchingUp)
}
if snap.Reason == "" {
t.Fatal("expected loop2 catching_up reason")
}
lastSnap, ok := manager.LastLoop2Snapshot()
if !ok {
t.Fatal("expected last loop2 snapshot")
}
if lastSnap.Mode != Loop2RuntimeModeCatchingUp {
t.Fatalf("last loop2 mode=%q, want %q", lastSnap.Mode, Loop2RuntimeModeCatchingUp)
}
}
func TestInProcessRuntimeManager_ObserveLoop2_NeedsRebuild(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "loop2-rebuild-b.blk")
assignB := masterv2.Assignment{
Name: "loop2-rebuild-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 5,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{assignB}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{VolumeName: "loop2-rebuild-vol", NodeID: "node-c"},
protocolv2.ReplicaSummaryResponse{
VolumeName: "loop2-rebuild-vol",
NodeID: "node-c",
Epoch: 5,
Role: "replica",
Mode: "needs_rebuild",
RoleApplied: true,
ReceiverReady: false,
CommittedLSN: 1,
DurableLSN: 1,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
)); err != nil {
t.Fatalf("register node-c target: %v", err)
}
snap, err := manager.ObserveLoop2("loop2-rebuild-vol", "node-b", 5)
if err != nil {
t.Fatalf("observe loop2 needs_rebuild: %v", err)
}
if snap.Mode != Loop2RuntimeModeNeedsRebuild {
t.Fatalf("loop2 mode=%q, want %q", snap.Mode, Loop2RuntimeModeNeedsRebuild)
}
if snap.Reason != "needs_rebuild" {
t.Fatalf("loop2 reason=%q, want needs_rebuild", snap.Reason)
}
perVolSnap, ok := manager.Loop2Snapshot("loop2-rebuild-vol")
if !ok {
t.Fatal("expected per-volume loop2 snapshot")
}
if perVolSnap.Mode != Loop2RuntimeModeNeedsRebuild {
t.Fatalf("per-volume loop2 mode=%q, want %q", perVolSnap.Mode, Loop2RuntimeModeNeedsRebuild)
}
}
func TestInProcessRuntimeManager_ExecuteReplicatedContinuity_HappyPath(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "continuity-b.blk")
pathC := filepath.Join(tempDir, "continuity-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "continuity-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "continuity-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "continuity-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
payload := bytes.Repeat([]byte{0x7A}, 4096)
result, err := manager.ExecuteReplicatedContinuity("continuity-vol", "node-b", 2, []string{"node-c"}, 0, payload)
if err != nil {
t.Fatalf("execute replicated continuity: %v", err)
}
if result.Loop2BeforeFailover.Mode != Loop2RuntimeModeKeepUp {
t.Fatalf("loop2 before failover=%q, want %q", result.Loop2BeforeFailover.Mode, Loop2RuntimeModeKeepUp)
}
if result.SelectedPrimaryNodeID != "node-c" {
t.Fatalf("selected primary=%q, want node-c", result.SelectedPrimaryNodeID)
}
if !result.DataMatch {
t.Fatalf("expected continuity data match: %+v", result)
}
if result.ReadBackLength != uint32(len(payload)) {
t.Fatalf("readback length=%d, want %d", result.ReadBackLength, len(payload))
}
}
func TestInProcessRuntimeManager_ExecuteReplicatedContinuity_GatedPath(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "continuity-gated-b.blk")
pathC := filepath.Join(tempDir, "continuity-gated-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "continuity-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "continuity-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "continuity-gated-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "continuity-gated-vol",
NodeID: "node-c",
Epoch: 2,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "continuity-gated-vol",
NodeID: "node-c",
Epoch: 2,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 1,
DurableLSN: 1,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
)); err != nil {
t.Fatalf("override node-c target: %v", err)
}
payload := bytes.Repeat([]byte{0x7B}, 4096)
result, err := manager.ExecuteReplicatedContinuity("continuity-gated-vol", "node-b", 2, []string{"node-c"}, 0, payload)
if err == nil {
t.Fatal("expected gated replicated continuity error")
}
if result.Loop2BeforeFailover.Mode != Loop2RuntimeModeNeedsRebuild {
t.Fatalf("loop2 before failover=%q, want %q", result.Loop2BeforeFailover.Mode, Loop2RuntimeModeNeedsRebuild)
}
if result.SelectedPrimaryNodeID != "" {
t.Fatalf("selected primary=%q, want empty on gated continuity", result.SelectedPrimaryNodeID)
}
if result.DataMatch {
t.Fatalf("unexpected continuity data match on gated path: %+v", result)
}
}
func TestInProcessRuntimeManager_RF2VolumeSurface_HealthyPackage(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "rf2-surface-b.blk")
pathC := filepath.Join(tempDir, "rf2-surface-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "rf2-surface-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "rf2-surface-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "rf2-surface-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
payload := bytes.Repeat([]byte{0x55}, 4096)
if _, err := manager.ExecuteReplicatedContinuity("rf2-surface-vol", "node-b", 2, []string{"node-c"}, 0, payload); err != nil {
t.Fatalf("execute replicated continuity: %v", err)
}
continuitySnap, ok := manager.ReplicatedContinuitySnapshot("rf2-surface-vol")
if !ok {
t.Fatal("expected continuity snapshot")
}
if continuitySnap.LastError != "" {
t.Fatalf("continuity last_error=%q, want empty", continuitySnap.LastError)
}
surface, ok := manager.RF2VolumeSurface("rf2-surface-vol")
if !ok {
t.Fatal("expected rf2 volume surface")
}
if !surface.HasLoop2 || !surface.HasFailover || !surface.HasContinuity {
t.Fatalf("surface source flags=%+v, want all true", surface)
}
if surface.Mode != RF2SurfaceModeHealthy {
t.Fatalf("surface mode=%q, want %q", surface.Mode, RF2SurfaceModeHealthy)
}
if surface.ReplicationMode != Loop2RuntimeModeKeepUp {
t.Fatalf("replication mode=%q, want %q", surface.ReplicationMode, Loop2RuntimeModeKeepUp)
}
if surface.FailoverStage != FailoverStageActivated {
t.Fatalf("failover stage=%q, want %q", surface.FailoverStage, FailoverStageActivated)
}
if surface.FailoverNodeID != "node-c" {
t.Fatalf("failover node=%q, want node-c", surface.FailoverNodeID)
}
if surface.ContinuityStatus != RF2ContinuityStatusProven {
t.Fatalf("continuity status=%q, want %q", surface.ContinuityStatus, RF2ContinuityStatusProven)
}
if surface.ContinuityNodeID != "node-c" {
t.Fatalf("continuity node=%q, want node-c", surface.ContinuityNodeID)
}
}
func TestInProcessRuntimeManager_RF2VolumeSurface_GatedPackage(t *testing.T) {
master := masterv2.New(masterv2.Config{})
manager, err := NewInProcessRuntimeManager(master)
if err != nil {
t.Fatalf("new runtime manager: %v", err)
}
nodeB, err := New(Config{NodeID: "node-b"})
if err != nil {
t.Fatalf("new node-b: %v", err)
}
defer nodeB.Close()
nodeC, err := New(Config{NodeID: "node-c"})
if err != nil {
t.Fatalf("new node-c: %v", err)
}
defer nodeC.Close()
if err := manager.RegisterNode(nodeB); err != nil {
t.Fatalf("register node-b: %v", err)
}
if err := manager.RegisterNode(nodeC); err != nil {
t.Fatalf("register node-c: %v", err)
}
tempDir := t.TempDir()
pathB := filepath.Join(tempDir, "rf2-surface-gated-b.blk")
pathC := filepath.Join(tempDir, "rf2-surface-gated-c.blk")
if err := master.DeclarePrimary(masterv2.VolumeSpec{
Name: "rf2-surface-gated-vol",
Path: pathB,
PrimaryNodeID: "node-a",
CreateOptions: testCreateOptions(),
}); err != nil {
t.Fatalf("declare primary: %v", err)
}
if err := nodeB.ApplyAssignments([]masterv2.Assignment{{
Name: "rf2-surface-gated-vol",
Path: pathB,
NodeID: "node-b",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-b: %v", err)
}
if err := nodeC.ApplyAssignments([]masterv2.Assignment{{
Name: "rf2-surface-gated-vol",
Path: pathC,
NodeID: "node-c",
Epoch: 2,
LeaseTTL: 30 * time.Second,
CreateOptions: testCreateOptions(),
Role: "primary",
}}); err != nil {
t.Fatalf("seed node-c: %v", err)
}
if err := manager.RegisterTarget(staticFailoverTarget(
"node-c",
masterv2.PromotionQueryResponse{
VolumeName: "rf2-surface-gated-vol",
NodeID: "node-c",
Epoch: 2,
CommittedLSN: 1,
WALHeadLSN: 1,
Eligible: false,
Reason: "needs_rebuild",
},
protocolv2.ReplicaSummaryResponse{
VolumeName: "rf2-surface-gated-vol",
NodeID: "node-c",
Epoch: 2,
Role: "replica",
Mode: "needs_rebuild",
CommittedLSN: 1,
DurableLSN: 1,
CheckpointLSN: 1,
RecoveryPhase: "needs_rebuild",
LastBarrierOK: false,
LastBarrierReason: "timeout",
Eligible: false,
Reason: "needs_rebuild",
},
)); err != nil {
t.Fatalf("override node-c target: %v", err)
}
payload := bytes.Repeat([]byte{0x56}, 4096)
if _, err := manager.ExecuteReplicatedContinuity("rf2-surface-gated-vol", "node-b", 2, []string{"node-c"}, 0, payload); err == nil {
t.Fatal("expected gated replicated continuity error")
}
continuitySnap, ok := manager.ReplicatedContinuitySnapshot("rf2-surface-gated-vol")
if !ok {
t.Fatal("expected continuity snapshot")
}
if continuitySnap.LastError == "" {
t.Fatal("expected continuity last_error")
}
surface, ok := manager.RF2VolumeSurface("rf2-surface-gated-vol")
if !ok {
t.Fatal("expected rf2 volume surface")
}
if surface.Mode != RF2SurfaceModeBlocked {
t.Fatalf("surface mode=%q, want %q", surface.Mode, RF2SurfaceModeBlocked)
}
if surface.ReplicationMode != Loop2RuntimeModeNeedsRebuild {
t.Fatalf("replication mode=%q, want %q", surface.ReplicationMode, Loop2RuntimeModeNeedsRebuild)
}
if surface.FailoverStage != FailoverStageFailed {
t.Fatalf("failover stage=%q, want %q", surface.FailoverStage, FailoverStageFailed)
}
if surface.FailoverError == "" {
t.Fatal("expected failover error")
}
if surface.ContinuityStatus != RF2ContinuityStatusFailed {
t.Fatalf("continuity status=%q, want %q", surface.ContinuityStatus, RF2ContinuityStatusFailed)
}
if surface.ContinuityError == "" {
t.Fatal("expected continuity error")
}
}
func mustHeartbeat(t *testing.T, node *Node) masterv2.NodeHeartbeat {
t.Helper()
hb, err := node.Heartbeat()
if err != nil {
t.Fatalf("heartbeat: %v", err)
}
return hb
}
func mustPromotionEvidence(t *testing.T, node *Node, volumeName string, epoch uint64) masterv2.PromotionQueryResponse {
t.Helper()
resp, err := node.QueryPromotionEvidence(masterv2.PromotionQueryRequest{
VolumeName: volumeName,
ExpectedEpoch: epoch,
})
if err != nil {
t.Fatalf("promotion evidence: %v", err)
}
return resp
}
func mustReplicaSummary(t *testing.T, node *Node, volumeName string, epoch uint64) protocolv2.ReplicaSummaryResponse {
t.Helper()
resp, err := node.QueryReplicaSummary(protocolv2.ReplicaSummaryRequest{
VolumeName: volumeName,
ExpectedEpoch: epoch,
})
if err != nil {
t.Fatalf("replica summary: %v", err)
}
return resp
}
func waitForCondition(t *testing.T, timeout time.Duration, cond func() bool) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("condition not reached before timeout")
}
func mustInProcessFailoverTarget(t *testing.T, node *Node) FailoverTarget {
t.Helper()
target, err := NewInProcessFailoverTarget(node)
if err != nil {
t.Fatalf("new in-process failover target: %v", err)
}
return target
}
func mustHybridFailoverTarget(t *testing.T, node *Node, transport FailoverEvidenceTransport) FailoverTarget {
t.Helper()
target, err := NewHybridInProcessFailoverTarget(node, transport)
if err != nil {
t.Fatalf("new hybrid failover target: %v", err)
}
return target
}
type staticReplicaSummarySource struct {
resp protocolv2.ReplicaSummaryResponse
err error
}
func (s staticReplicaSummarySource) QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) {
return s.resp, s.err
}
type staticFailoverAdapter struct {
promotion masterv2.PromotionQueryResponse
summary protocolv2.ReplicaSummaryResponse
err error
}
func staticFailoverTarget(nodeID string, promotion masterv2.PromotionQueryResponse, summary protocolv2.ReplicaSummaryResponse) FailoverTarget {
adapter := staticFailoverAdapter{
promotion: promotion,
summary: summary,
}
return FailoverTarget{
NodeID: nodeID,
Evidence: adapter,
Takeover: adapter,
}
}
func (s staticFailoverAdapter) QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) {
return s.promotion, s.err
}
func (s staticFailoverAdapter) QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) {
return s.summary, s.err
}
func (s staticFailoverAdapter) PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) {
return ReconstructedPrimaryTruth{}, fmt.Errorf("static failover participant cannot prepare takeover")
}
func (s staticFailoverAdapter) GatePrimaryActivation(string, ReconstructedPrimaryTruth) error {
return fmt.Errorf("static failover participant cannot gate activation")
}
func testCreateOptions() blockvol.CreateOptions {
return blockvol.CreateOptions{
VolumeSize: 1 * 1024 * 1024,
BlockSize: 4096,
WALSize: 256 * 1024,
}
}
Reference in New Issue View Git Blame Copy Permalink