From b8c6944e3f8e7701136e62ad8ee957e80c7589a9 Mon Sep 17 00:00:00 2001 From: pingqiu Date: Sun, 5 Apr 2026 13:08:02 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20V2=20MVP=20milestone=20=E2=80=94=20mast?= =?UTF-8?q?erv2=20+=20volumev2=20+=20in-process=20failover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V2 runtime packages: - sw-block/runtime/masterv2: identity authority (desired state, heartbeat handling, promotion arbitration via SelectPromotionCandidate) - sw-block/runtime/volumev2: per-volume micro-cluster shell (node, orchestrator, control session, iSCSI frontend, takeover gate, failover session + driver, replica summary reconstruction) - sw-block/runtime/purev2: RF1 execution shell (engine + store + dispatcher + local boundary observations) - sw-block/runtime/protocolv2: three-channel separation (heartbeat/assignment/query + replica summary) V2 binaries: - sw-block/cmd/v2singleblock: single-node RF1 block server - sw-block/cmd/purev2rf1: minimal RF1 runtime binary Milestone capabilities: - RF1 write/read/sync with engine-driven mode projection - masterv2 ↔ volumev2 heartbeat convergence + assignment reissue - Promotion query with fresh CommittedLSN/WALHeadLSN evidence - Replica summary for bounded takeover reconstruction - Primary-loss reconstruction from peer summaries (fail-closed gate) - In-process failover driver with session observability - Local boundary observations feed engine (Committed/Durable/Checkpoint) Design docs: - v2-two-loop-protocol.md: identity vs data-control separation - v2-automata-ownership-map.md: event/command ownership split - v2-loop1-surface-draft.md: heartbeat/query/assignment field spec - v2-volumev2-single-node-mvp.md: target layering - v2-kernel-closure-review.md: per-volume micro-cluster principle - v2-pure-runtime-rf1-bootstrap.md, v2-capability-map.md, v2-proof-and-retest-pyramid.md Co-Authored-By: Claude Opus 4.6 (1M context) --- sw-block/cmd/purev2rf1/main.go | 110 ++ sw-block/cmd/v2singleblock/main.go | 436 ++++++ sw-block/cmd/v2singleblock/main_test.go | 129 ++ sw-block/design/v2-automata-ownership-map.md | 339 +++++ sw-block/design/v2-capability-map.md | 474 +++++++ sw-block/design/v2-kernel-closure-review.md | 143 ++ sw-block/design/v2-loop1-surface-draft.md | 253 ++++ .../design/v2-proof-and-retest-pyramid.md | 186 +++ .../design/v2-pure-runtime-rf1-bootstrap.md | 148 ++ sw-block/design/v2-two-loop-protocol.md | 414 ++++++ .../design/v2-volumev2-single-node-mvp.md | 151 ++ sw-block/runtime/masterv2/master.go | 264 ++++ sw-block/runtime/masterv2/master_test.go | 138 ++ sw-block/runtime/protocolv2/identity.go | 68 + sw-block/runtime/protocolv2/recovery.go | 33 + sw-block/runtime/purev2/doc.go | 4 + sw-block/runtime/purev2/runtime.go | 349 +++++ sw-block/runtime/purev2/runtime_test.go | 154 ++ sw-block/runtime/volumev2/control_session.go | 45 + sw-block/runtime/volumev2/dataplane.go | 83 ++ sw-block/runtime/volumev2/failover.go | 305 ++++ sw-block/runtime/volumev2/failover_driver.go | 122 ++ sw-block/runtime/volumev2/frontend.go | 92 ++ sw-block/runtime/volumev2/frontend_test.go | 233 ++++ sw-block/runtime/volumev2/orchestrator.go | 37 + sw-block/runtime/volumev2/poc_test.go | 1242 +++++++++++++++++ sw-block/runtime/volumev2/reconstruction.go | 150 ++ .../runtime/volumev2/reconstruction_test.go | 138 ++ sw-block/runtime/volumev2/takeover.go | 115 ++ sw-block/runtime/volumev2/volume.go | 312 +++++ 30 files changed, 6667 insertions(+) create mode 100644 sw-block/cmd/purev2rf1/main.go create mode 100644 sw-block/cmd/v2singleblock/main.go create mode 100644 sw-block/cmd/v2singleblock/main_test.go create mode 100644 sw-block/design/v2-automata-ownership-map.md create mode 100644 sw-block/design/v2-capability-map.md create mode 100644 sw-block/design/v2-kernel-closure-review.md create mode 100644 sw-block/design/v2-loop1-surface-draft.md create mode 100644 sw-block/design/v2-proof-and-retest-pyramid.md create mode 100644 sw-block/design/v2-pure-runtime-rf1-bootstrap.md create mode 100644 sw-block/design/v2-two-loop-protocol.md create mode 100644 sw-block/design/v2-volumev2-single-node-mvp.md create mode 100644 sw-block/runtime/masterv2/master.go create mode 100644 sw-block/runtime/masterv2/master_test.go create mode 100644 sw-block/runtime/protocolv2/identity.go create mode 100644 sw-block/runtime/protocolv2/recovery.go create mode 100644 sw-block/runtime/purev2/doc.go create mode 100644 sw-block/runtime/purev2/runtime.go create mode 100644 sw-block/runtime/purev2/runtime_test.go create mode 100644 sw-block/runtime/volumev2/control_session.go create mode 100644 sw-block/runtime/volumev2/dataplane.go create mode 100644 sw-block/runtime/volumev2/failover.go create mode 100644 sw-block/runtime/volumev2/failover_driver.go create mode 100644 sw-block/runtime/volumev2/frontend.go create mode 100644 sw-block/runtime/volumev2/frontend_test.go create mode 100644 sw-block/runtime/volumev2/orchestrator.go create mode 100644 sw-block/runtime/volumev2/poc_test.go create mode 100644 sw-block/runtime/volumev2/reconstruction.go create mode 100644 sw-block/runtime/volumev2/reconstruction_test.go create mode 100644 sw-block/runtime/volumev2/takeover.go create mode 100644 sw-block/runtime/volumev2/volume.go diff --git a/sw-block/cmd/purev2rf1/main.go b/sw-block/cmd/purev2rf1/main.go new file mode 100644 index 000000000..52c397173 --- /dev/null +++ b/sw-block/cmd/purev2rf1/main.go @@ -0,0 +1,110 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/purev2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + switch os.Args[1] { + case "bootstrap": + if err := runBootstrap(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + case "status": + if err := runStatus(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + default: + usage() + os.Exit(2) + } +} + +func runBootstrap(args []string) error { + fs := flag.NewFlagSet("bootstrap", flag.ContinueOnError) + path := fs.String("path", "", "block volume path") + sizeBytes := fs.Uint64("size-bytes", 1*1024*1024, "logical volume size in bytes") + blockSize := fs.Uint("block-size", 4096, "block size in bytes") + walSize := fs.Uint64("wal-size", 256*1024, "wal size in bytes") + epoch := fs.Uint64("epoch", 1, "assignment epoch") + leaseMs := fs.Int("lease-ms", 30000, "lease ttl in milliseconds") + if err := fs.Parse(args); err != nil { + return err + } + if *path == "" { + return fmt.Errorf("bootstrap: --path is required") + } + + rt := purev2.New(purev2.Config{}) + defer rt.Close() + + opts := blockvol.CreateOptions{ + VolumeSize: *sizeBytes, + BlockSize: uint32(*blockSize), + WALSize: *walSize, + } + if err := rt.BootstrapPrimary(*path, opts, *epoch, time.Duration(*leaseMs)*time.Millisecond); err != nil { + return err + } + snap, err := rt.Snapshot(*path) + if err != nil { + return err + } + return printJSON(snap) +} + +func runStatus(args []string) error { + fs := flag.NewFlagSet("status", flag.ContinueOnError) + path := fs.String("path", "", "block volume path") + epoch := fs.Uint64("epoch", 0, "optional epoch to rebuild core projection") + leaseMs := fs.Int("lease-ms", 30000, "lease ttl in milliseconds") + if err := fs.Parse(args); err != nil { + return err + } + if *path == "" { + return fmt.Errorf("status: --path is required") + } + + rt := purev2.New(purev2.Config{}) + defer rt.Close() + + if err := rt.OpenVolume(*path); err != nil { + return err + } + if *epoch > 0 { + if err := rt.ApplyPrimaryAssignment(*path, *epoch, time.Duration(*leaseMs)*time.Millisecond); err != nil { + return err + } + } + snap, err := rt.Snapshot(*path) + if err != nil { + return err + } + return printJSON(snap) +} + +func printJSON(v any) error { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage:") + fmt.Fprintln(os.Stderr, " purev2rf1 bootstrap --path [--size-bytes N --block-size N --wal-size N --epoch N]") + fmt.Fprintln(os.Stderr, " purev2rf1 status --path [--epoch N]") +} diff --git a/sw-block/cmd/v2singleblock/main.go b/sw-block/cmd/v2singleblock/main.go new file mode 100644 index 000000000..698b73254 --- /dev/null +++ b/sw-block/cmd/v2singleblock/main.go @@ -0,0 +1,436 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "os" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/volumev2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" +) + +type smokeResult struct { + VolumeName string `json:"volume_name"` + Path string `json:"path"` + NodeID string `json:"node_id"` + Epoch uint64 `json:"epoch"` + Role string `json:"role"` + Mode string `json:"mode"` + Reason string `json:"reason"` + Readback string `json:"readback_hex"` +} + +type restartSmokeResult struct { + VolumeName string `json:"volume_name"` + Path string `json:"path"` + NodeID string `json:"node_id"` + Epoch uint64 `json:"epoch"` + Role string `json:"role"` + Mode string `json:"mode"` + Reason string `json:"reason"` + InitialWrite string `json:"initial_write_hex"` + PostRestart string `json:"post_restart_hex"` +} + +type iscsiSmokeResult struct { + VolumeName string `json:"volume_name"` + Path string `json:"path"` + NodeID string `json:"node_id"` + Epoch uint64 `json:"epoch"` + Role string `json:"role"` + Mode string `json:"mode"` + Reason string `json:"reason"` + IQN string `json:"iqn"` + Address string `json:"address"` +} + +type commonFlags struct { + name string + path string + nodeID string + writeText string + sizeBytes uint64 + blockSize uint + walSize uint64 +} + +type singleNodeEnv struct { + master *masterv2.Master + node *volumev2.Node + orchestrator *volumev2.Orchestrator +} + +var commandOutput io.Writer = os.Stdout + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + switch os.Args[1] { + case "smoke": + if err := runSmoke(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + case "restart-smoke": + if err := runRestartSmoke(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + case "iscsi-smoke": + if err := runISCSISmoke(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + default: + usage() + os.Exit(2) + } +} + +func runSmoke(args []string) error { + cfg, err := parseCommonFlags("smoke", args) + if err != nil { + return err + } + if cfg.path == "" { + return fmt.Errorf("smoke: --path is required") + } + + env, err := bootstrapSingleNode(cfg) + if err != nil { + return err + } + defer env.close() + + payload := paddedPayload([]byte(cfg.writeText), uint32(cfg.blockSize)) + if err := env.node.WriteLBA(cfg.name, 0, payload); err != nil { + return err + } + if err := env.node.SyncCache(cfg.name); err != nil { + return err + } + readBack, err := env.node.ReadLBA(cfg.name, 0, uint32(len(payload))) + if err != nil { + return err + } + snap, err := env.node.Snapshot(cfg.name) + if err != nil { + return err + } + + result := smokeResult{ + VolumeName: cfg.name, + Path: cfg.path, + NodeID: cfg.nodeID, + Epoch: snap.Status.Epoch, + Role: snap.Status.Role.String(), + Readback: hex.EncodeToString(readBack[:len(payload)]), + } + if snap.HasProjection { + result.Mode = string(snap.Projection.Mode.Name) + result.Reason = snap.Projection.Publication.Reason + } + return printJSON(result) +} + +func runRestartSmoke(args []string) error { + cfg, err := parseCommonFlags("restart-smoke", args) + if err != nil { + return err + } + if cfg.path == "" { + return fmt.Errorf("restart-smoke: --path is required") + } + + payload := paddedPayload([]byte(cfg.writeText), uint32(cfg.blockSize)) + master := masterv2.New(masterv2.Config{LeaseTTL: 30 * time.Second}) + if err := declarePrimary(master, cfg); err != nil { + return err + } + + func() { + node, nodeErr := volumev2.New(volumev2.Config{NodeID: cfg.nodeID}) + if nodeErr != nil { + err = nodeErr + return + } + defer node.Close() + session, nodeErr := volumev2.NewInProcessSession(master) + if nodeErr != nil { + err = nodeErr + return + } + orchestrator, nodeErr := volumev2.NewOrchestrator(node, session) + if nodeErr != nil { + err = nodeErr + return + } + if nodeErr = syncTwice(orchestrator); nodeErr != nil { + err = nodeErr + return + } + if nodeErr = node.WriteLBA(cfg.name, 0, payload); nodeErr != nil { + err = nodeErr + return + } + if nodeErr = node.SyncCache(cfg.name); nodeErr != nil { + err = nodeErr + return + } + }() + if err != nil { + return err + } + + node, err := volumev2.New(volumev2.Config{NodeID: cfg.nodeID}) + if err != nil { + return err + } + defer node.Close() + session, err := volumev2.NewInProcessSession(master) + if err != nil { + return err + } + orchestrator, err := volumev2.NewOrchestrator(node, session) + if err != nil { + return err + } + if err := syncTwice(orchestrator); err != nil { + return err + } + readBack, err := node.ReadLBA(cfg.name, 0, uint32(len(payload))) + if err != nil { + return err + } + snap, err := node.Snapshot(cfg.name) + if err != nil { + return err + } + + result := restartSmokeResult{ + VolumeName: cfg.name, + Path: cfg.path, + NodeID: cfg.nodeID, + Epoch: snap.Status.Epoch, + Role: snap.Status.Role.String(), + InitialWrite: hex.EncodeToString(payload), + PostRestart: hex.EncodeToString(readBack[:len(payload)]), + } + if snap.HasProjection { + result.Mode = string(snap.Projection.Mode.Name) + result.Reason = snap.Projection.Publication.Reason + } + return printJSON(result) +} + +func runISCSISmoke(args []string) error { + fs := flag.NewFlagSet("iscsi-smoke", flag.ContinueOnError) + name := fs.String("name", "single-node-vol", "logical volume name") + path := fs.String("path", "", "block volume file path") + nodeID := fs.String("node", "node-a", "volumev2 node id") + writeText := fs.String("write-text", "v2-single-node-smoke", "payload written at LBA 0") + sizeBytes := fs.Uint64("size-bytes", 1*1024*1024, "logical volume size in bytes") + blockSize := fs.Uint("block-size", 4096, "block size in bytes") + walSize := fs.Uint64("wal-size", 256*1024, "wal size in bytes") + listenAddr := fs.String("listen-addr", "127.0.0.1:0", "iSCSI target listen address") + iqn := fs.String("iqn", "", "target iqn") + if err := fs.Parse(args); err != nil { + return err + } + cfg := commonFlags{ + name: *name, + path: *path, + nodeID: *nodeID, + writeText: *writeText, + sizeBytes: *sizeBytes, + blockSize: *blockSize, + walSize: *walSize, + } + if cfg.path == "" { + return fmt.Errorf("iscsi-smoke: --path is required") + } + + env, err := bootstrapSingleNode(cfg) + if err != nil { + return err + } + defer env.close() + + export, err := env.node.ExportISCSI(cfg.name, *listenAddr, *iqn) + if err != nil { + return err + } + defer export.Close() + + if err := performISCSILogin(export.Address(), export.IQN()); err != nil { + return err + } + snap, err := env.node.Snapshot(cfg.name) + if err != nil { + return err + } + + result := iscsiSmokeResult{ + VolumeName: cfg.name, + Path: cfg.path, + NodeID: cfg.nodeID, + Epoch: snap.Status.Epoch, + Role: snap.Status.Role.String(), + IQN: export.IQN(), + Address: export.Address(), + } + if snap.HasProjection { + result.Mode = string(snap.Projection.Mode.Name) + result.Reason = snap.Projection.Publication.Reason + } + return printJSON(result) +} + +func parseCommonFlags(name string, args []string) (commonFlags, error) { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + cfg := commonFlags{} + namePtr := fs.String("name", "single-node-vol", "logical volume name") + pathPtr := fs.String("path", "", "block volume file path") + nodePtr := fs.String("node", "node-a", "volumev2 node id") + writePtr := fs.String("write-text", "v2-single-node-smoke", "payload written at LBA 0") + sizePtr := fs.Uint64("size-bytes", 1*1024*1024, "logical volume size in bytes") + blockPtr := fs.Uint("block-size", 4096, "block size in bytes") + walPtr := fs.Uint64("wal-size", 256*1024, "wal size in bytes") + if err := fs.Parse(args); err != nil { + return cfg, err + } + cfg.name = *namePtr + cfg.path = *pathPtr + cfg.nodeID = *nodePtr + cfg.writeText = *writePtr + cfg.sizeBytes = *sizePtr + cfg.blockSize = *blockPtr + cfg.walSize = *walPtr + return cfg, nil +} + +func bootstrapSingleNode(cfg commonFlags) (*singleNodeEnv, error) { + master := masterv2.New(masterv2.Config{LeaseTTL: 30 * time.Second}) + if err := declarePrimary(master, cfg); err != nil { + return nil, err + } + node, err := volumev2.New(volumev2.Config{NodeID: cfg.nodeID}) + if err != nil { + return nil, err + } + session, err := volumev2.NewInProcessSession(master) + if err != nil { + node.Close() + return nil, err + } + orchestrator, err := volumev2.NewOrchestrator(node, session) + if err != nil { + node.Close() + return nil, err + } + if err := syncTwice(orchestrator); err != nil { + node.Close() + return nil, err + } + return &singleNodeEnv{ + master: master, + node: node, + orchestrator: orchestrator, + }, nil +} + +func declarePrimary(master *masterv2.Master, cfg commonFlags) error { + return master.DeclarePrimary(masterv2.VolumeSpec{ + Name: cfg.name, + Path: cfg.path, + PrimaryNodeID: cfg.nodeID, + CreateOptions: blockvol.CreateOptions{ + VolumeSize: cfg.sizeBytes, + BlockSize: uint32(cfg.blockSize), + WALSize: cfg.walSize, + }, + }) +} + +func (env *singleNodeEnv) close() { + if env == nil || env.node == nil { + return + } + env.node.Close() +} + +func syncTwice(orchestrator *volumev2.Orchestrator) error { + if err := orchestrator.SyncOnce(); err != nil { + return err + } + return orchestrator.SyncOnce() +} + +func performISCSILogin(addr, targetIQN string) error { + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + return fmt.Errorf("dial iscsi target %s: %w", addr, err) + } + defer conn.Close() + + params := iscsi.NewParams() + params.Set("InitiatorName", "iqn.2026-04.com.seaweedfs:cli.initiator") + params.Set("TargetName", targetIQN) + params.Set("SessionType", "Normal") + + loginReq := &iscsi.PDU{} + loginReq.SetOpcode(iscsi.OpLoginReq) + loginReq.SetLoginStages(iscsi.StageSecurityNeg, iscsi.StageFullFeature) + loginReq.SetLoginTransit(true) + loginReq.SetISID([6]byte{0x00, 0x02, 0x3D, 0x00, 0x00, 0x02}) + loginReq.SetCmdSN(1) + loginReq.DataSegment = params.Encode() + + if err := iscsi.WritePDU(conn, loginReq); err != nil { + return fmt.Errorf("write iscsi login: %w", err) + } + resp, err := iscsi.ReadPDU(conn) + if err != nil { + return fmt.Errorf("read iscsi login: %w", err) + } + if resp.LoginStatusClass() != iscsi.LoginStatusSuccess { + return fmt.Errorf("iscsi login failed: %d/%d", resp.LoginStatusClass(), resp.LoginStatusDetail()) + } + return nil +} + +func paddedPayload(in []byte, blockSize uint32) []byte { + size := int(blockSize) + if size <= 0 { + size = len(in) + } + if size < len(in) { + size = len(in) + } + out := make([]byte, size) + copy(out, in) + return out +} + +func printJSON(v any) error { + enc := json.NewEncoder(commandOutput) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage:") + fmt.Fprintln(os.Stderr, " v2singleblock smoke --path [--name N --node N --write-text TEXT]") + fmt.Fprintln(os.Stderr, " v2singleblock restart-smoke --path [--name N --node N --write-text TEXT]") + fmt.Fprintln(os.Stderr, " v2singleblock iscsi-smoke --path [--name N --node N --iqn IQN]") +} diff --git a/sw-block/cmd/v2singleblock/main_test.go b/sw-block/cmd/v2singleblock/main_test.go new file mode 100644 index 000000000..d4f593a8b --- /dev/null +++ b/sw-block/cmd/v2singleblock/main_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "bytes" + "encoding/json" + "path/filepath" + "strings" + "testing" +) + +func TestRunSmoke_ProducesSingleNodeMVPResult(t *testing.T) { + path := filepath.Join(t.TempDir(), "single-node-mvp.blk") + + var out bytes.Buffer + oldOutput := commandOutput + commandOutput = &out + defer func() { + commandOutput = oldOutput + }() + + runErr := runSmoke([]string{ + "--path", path, + "--name", "mvp-vol", + "--node", "node-a", + "--write-text", "hello-v2", + }) + if runErr != nil { + t.Fatalf("runSmoke: %v", runErr) + } + + var result smokeResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("unmarshal output %q: %v", out.String(), err) + } + if result.VolumeName != "mvp-vol" { + t.Fatalf("volume_name=%q", result.VolumeName) + } + if result.Role != "primary" { + t.Fatalf("role=%q", result.Role) + } + if result.Mode != "allocated_only" { + t.Fatalf("mode=%q", result.Mode) + } + if !strings.Contains(result.Readback, "68656c6c6f2d7632") { + t.Fatalf("readback_hex=%q", result.Readback) + } +} + +func TestRunRestartSmoke_PreservesDataAcrossRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "single-node-restart.blk") + + var out bytes.Buffer + oldOutput := commandOutput + commandOutput = &out + defer func() { + commandOutput = oldOutput + }() + + runErr := runRestartSmoke([]string{ + "--path", path, + "--name", "restart-vol", + "--node", "node-a", + "--write-text", "hello-restart", + }) + if runErr != nil { + t.Fatalf("runRestartSmoke: %v", runErr) + } + + var result restartSmokeResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("unmarshal output %q: %v", out.String(), err) + } + if result.VolumeName != "restart-vol" { + t.Fatalf("volume_name=%q", result.VolumeName) + } + if result.Role != "primary" { + t.Fatalf("role=%q", result.Role) + } + if result.Mode != "allocated_only" { + t.Fatalf("mode=%q", result.Mode) + } + if result.InitialWrite != result.PostRestart { + t.Fatalf("initial=%q post_restart=%q", result.InitialWrite, result.PostRestart) + } + if !strings.Contains(result.PostRestart, "68656c6c6f2d72657374617274") { + t.Fatalf("post_restart_hex=%q", result.PostRestart) + } +} + +func TestRunISCSISmoke_ExportsFrontendAndReportsAddress(t *testing.T) { + path := filepath.Join(t.TempDir(), "single-node-iscsi.blk") + + var out bytes.Buffer + oldOutput := commandOutput + commandOutput = &out + defer func() { + commandOutput = oldOutput + }() + + runErr := runISCSISmoke([]string{ + "--path", path, + "--name", "iscsi-vol", + "--node", "node-a", + "--iqn", "iqn.2026-04.com.seaweedfs:test.cli", + }) + if runErr != nil { + t.Fatalf("runISCSISmoke: %v", runErr) + } + + var result iscsiSmokeResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("unmarshal output %q: %v", out.String(), err) + } + if result.VolumeName != "iscsi-vol" { + t.Fatalf("volume_name=%q", result.VolumeName) + } + if result.Role != "primary" { + t.Fatalf("role=%q", result.Role) + } + if result.Mode != "allocated_only" { + t.Fatalf("mode=%q", result.Mode) + } + if result.IQN != "iqn.2026-04.com.seaweedfs:test.cli" { + t.Fatalf("iqn=%q", result.IQN) + } + if !strings.Contains(result.Address, "127.0.0.1:") { + t.Fatalf("address=%q", result.Address) + } +} diff --git a/sw-block/design/v2-automata-ownership-map.md b/sw-block/design/v2-automata-ownership-map.md new file mode 100644 index 000000000..ead1b513b --- /dev/null +++ b/sw-block/design/v2-automata-ownership-map.md @@ -0,0 +1,339 @@ +# V2 Automata Ownership Map + +Date: 2026-04-05 +Status: active + +## Purpose + +This note translates the two-loop protocol into automata ownership. + +The goal is to answer a practical question for the next implementation step: + +1. which decisions belong to `masterv2` +2. which decisions belong to the primary-side data-control automata +3. which current V2 events and commands already fit that split +4. which surfaces should remain compressed projections only + +## Top-Level Split + +There are four different layers of meaning: + +1. identity authority +2. identity evidence +3. data-control truth +4. outward compressed projection + +They must not be merged. + +## Operating Model + +One volume should be treated as a micro-cluster: + +1. `masterv2` is outside the micro-cluster and grants identity authority +2. the selected primary is inside the micro-cluster and owns data-control truth +3. replicas contribute bounded evidence and execution progress to the primary + +This means `masterv2` is allowed to authorize takeover, but not to act as the +continuous recovery planner. + +```mermaid +flowchart TD + master[masterv2] + assignment[AssignmentChannel] + heartbeat[HeartbeatChannel] + query[PromotionQueryChannel] + primary[PrimaryDataControlAutomata] + replica[ReplicaDataControlAutomata] + projection[OutwardProjection] + + master --> assignment + assignment --> primary + assignment --> replica + primary --> projection + replica --> projection + projection --> heartbeat + master --> query + query --> primary + query --> replica + primary --> replica + replica --> primary +``` + +## Channel Ownership + +### 1. Heartbeat + +Heartbeat is not a recovery-planning channel. + +Heartbeat should answer only: + +1. is the node alive +2. what role and epoch has been applied +3. what compressed mode is outwardly visible +4. whether replica receiver readiness is present + +Heartbeat should not answer: + +1. which LSN to catch up to +2. whether a reconnect gap is recoverable +3. which source to rebuild from +4. per-replica durable progress + +### 2. Promotion Query + +Promotion query is the only `masterv2` path that may request fresh promotion +evidence. + +It should answer only: + +1. what is your current `CommittedLSN` +2. what is your current `WALHeadLSN` +3. which `Epoch` do you recognize +4. what `Role` do you believe you have +5. are you eligible to become primary + +Promotion query should not answer: + +1. replay plan details +2. full session graph +3. retention budget internals +4. detailed catch-up progress + +### 3. Assignment + +Assignment is the identity authorization channel. + +It should answer only: + +1. who is primary +2. who is replica +3. which epoch is now active +4. which replica identities and addresses belong to the set +5. whether rebuilding role is assigned + +Assignment should not answer: + +1. committed boundary +2. durable boundary +3. catch-up target LSN +4. detailed recovery plan + +### 3a. Takeover Authorization Versus Takeover Choreography + +This distinction should stay explicit: + +1. `masterv2` may authorize a replacement primary +2. the replacement primary must reconstruct bounded truth itself +3. the replacement primary must decide whether takeover is safe, degraded, or + rebuild-only + +So: + +1. takeover authorization belongs to `Assignment` +2. takeover reconstruction belongs to `Loop 2` +3. detailed recovery sequencing belongs to the primary-led automata + +### 4. Loop 2 Data Control + +Loop 2 is where data-control truth lives. + +It should answer: + +1. what is committed +2. what is durable +3. whether barrier lineage is healthy +4. whether a replica is in `keepup`, `catchup`, `degraded`, or `needs_rebuild` +5. whether rebuild is currently the only safe path + +## Existing V2 Events Mapped To Owners + +### Identity-Control Entry Event + +These belong to assignment application or identity evidence: + +1. `AssignmentDelivered` +2. `RoleApplied` +3. `ReceiverReadyObserved` + +Interpretation: + +- `AssignmentDelivered` enters from `Assignment` +- `RoleApplied` is local confirmation after identity application +- `ReceiverReadyObserved` is local evidence that may affect compressed outward mode + +### Data-Control Events + +These belong to the primary-led data-control automata: + +1. `ShipperConfiguredObserved` +2. `ShipperConnectedObserved` +3. `DiagnosticShippedAdvanced` +4. `CommittedLSNAdvanced` +5. `BarrierAccepted` +6. `BarrierRejected` +7. `CheckpointAdvanced` +8. `CatchUpPlanned` +9. `RecoveryProgressObserved` +10. `CatchUpCompleted` +11. `NeedsRebuildObserved` +12. `RebuildStarted` +13. `RebuildCommitted` + +Interpretation: + +- these events must stay inside `Loop 2` +- they may influence outward `Mode` +- they must not be surfaced to `masterv2` as raw automata state + +## Existing V2 Commands Mapped To Owners + +### Identity Commands + +These belong to assignment realization: + +1. `ApplyRoleCommand` +2. `StartReceiverCommand` + +These are still executed locally, but they exist to realize identity control. + +### Data-Control Commands + +These belong to the primary-led data-control automata: + +1. `ConfigureShipperCommand` +2. `StartRecoveryTaskCommand` +3. `DrainRecoveryTaskCommand` +4. `StartCatchUpCommand` +5. `StartRebuildCommand` +6. `InvalidateSessionCommand` + +Interpretation: + +- these commands must not be emitted by `masterv2` +- these commands are consequences of data-control truth + +### Projection Command + +`PublishProjectionCommand` is neither identity truth nor data-control truth. + +It is the compressed outward surface emitted from the core. + +This command exists so: + +1. heartbeat can expose compressed state +2. debug and product surfaces can stay consistent +3. runtime seams do not invent independent meanings + +## Logic Judgments + +This is the decision table that should guide future changes. + +### Judgment: Is this enough for heartbeat? + +If a fact is needed only to answer: + +1. alive +2. applied role +3. outward mode + +then it belongs in heartbeat projection. + +If it is needed to answer: + +1. which node is safest to promote +2. whether recovery is catch-up or rebuild +3. how far durability actually advanced + +then it does not belong in normal heartbeat. + +### Judgment: Does master need this continuously? + +If a fact affects: + +1. assignment ownership +2. lease ownership +3. fencing +4. failover authorization + +then `masterv2` may need it. + +If a fact affects: + +1. replay planning +2. retention decisions +3. barrier lineage +4. rebuild execution + +then `masterv2` must not own it continuously. + +### Judgment: Is master authorizing or choreographing? + +If the action is: + +1. choose a legal owner +2. fence stale owners +3. publish epoch and replica identity + +then it belongs to `masterv2`. + +If the action is: + +1. reconstruct bounded truth from summaries +2. decide `keepup` vs `catchup` +3. select rebuild source and progress target +4. gate takeover on degraded or ambiguous lineage + +then it belongs to the selected primary, not `masterv2`. + +### Judgment: Does this belong to promotion query? + +A fact belongs to promotion query when all are true: + +1. it is needed only during promotion arbitration +2. stale cached heartbeat would be unsafe +3. the fact is still smaller than full recovery truth + +Examples: + +1. `CommittedLSN` +2. `WALHeadLSN` +3. `Epoch` +4. promotion eligibility + +### Judgment: Does this belong to Loop 2? + +A fact belongs to `Loop 2` when it changes at write/barrier/reconnect scale or +when it directly affects recovery planning. + +Examples: + +1. `ReplicaFlushedLSN` +2. `CatchUpTarget` +3. retention floor +4. session invalidation +5. recovery progress + +## What Should Change Next + +Before any major implementation, these changes should be reflected in code +structure: + +1. `masterv2` types should split heartbeat, query, and assignment surfaces +2. `volumev2` control session should stop pretending heartbeat and promotion + evidence are the same message shape +3. engine-facing runtime seams should explicitly tag which observations are + identity evidence and which are data-control observations +4. tests should separately prove: + - heartbeat convergence + - promotion query freshness + - data-control progress closure + +## Non-Goals + +This note does not freeze every field of the future wire protocol. + +It only fixes: + +1. ownership +2. judgment boundaries +3. which existing V2 automata pieces are already valid diff --git a/sw-block/design/v2-capability-map.md b/sw-block/design/v2-capability-map.md new file mode 100644 index 000000000..ba935d7a1 --- /dev/null +++ b/sw-block/design/v2-capability-map.md @@ -0,0 +1,474 @@ +# V2 Capability Map + +Date: 2026-04-05 +Status: active +Purpose: define the V2 capability expansion map that drives feature closure, test closure, and the transition from bounded scenario debugging to systematic product validation + +## Why This Document Exists + +If `V2` is a real system line, it needs more than: + +1. accepted protocol truths +2. passing point fixes +3. a few successful scenarios + +It also needs one explicit map that answers: + +1. what product capabilities exist in the V2 line +2. in what order those capabilities should close +3. what "done" means for each capability +4. which tests prove the capability +5. which proofs are V2-owned versus runtime-specific + +This document is that map. + +It complements: + +1. `v2-protocol-truths.md` for stable semantic rules +2. `v2-product-completion-overview.md` for product-level completion status +3. `v2-phase-development-plan.md` for active execution sequencing +4. `v2_scenarios.md` for scenario backlog and historical failure sources + +## How To Use This Map + +For any new feature, bug fix, or test expansion, ask: + +1. which capability tier does this belong to +2. which closure claim does it strengthen +3. which proof tier should carry it +4. whether it is V2-owned truth or current-runtime integration + +This prevents three common failures: + +1. growing V2 by random scenario accumulation +2. confusing `weed` integration success with V2 semantic completion +3. re-testing everything from zero when the runtime boundary changes later + +## Core Method + +The map uses three linked ideas: + +### 1. Capability expansion + +V2 should expand from: + +1. single-volume correctness +2. bounded RF=2 replication +3. failover and rejoin +4. multi-replica behavior +5. lifecycle operations +6. control-plane and operations closure +7. CSI and product-surface closure + +### 2. Completion definition + +A capability is not "done" because code exists. + +It is only closed when all of these are true: + +1. semantic rule is explicit +2. runtime path exists +3. observability exists +4. focused tests prove the rule +5. one product-level scenario proves the real path + +### 3. Proof layering + +Each capability should be proven across four proof tiers: + +1. `Core semantic` + - pure V2 truth + - fastest feedback + - should remain reusable if runtime changes +2. `Seam / adapter` + - queue, heartbeat, registry, proto, assignment, bridge ownership + - catches most integrated bugs cheaply +3. `Integrated runtime` + - real `weed` path today + - smaller number of high-value scenarios +4. `Soak / benchmark / adversarial` + - slow, broad, or disturbance-heavy validation + - not the daily development loop + +## Capability Tiers + +## Tier 0: Semantic Foundation + +Goal: + +1. make V2 the source of truth for replication semantics + +Main closure claims: + +1. epoch and lineage are authoritative +2. committed truth is explicit +3. catch-up versus rebuild boundary is explicit +4. stale authority fails closed +5. replica identity is stable across endpoint change + +Done means: + +1. truths are explicit in `v2-protocol-truths.md` +2. engine events and commands preserve those truths +3. core tests cover replay, stale events, fencing, and recovery choice + +Primary proof tiers: + +1. core semantic +2. seam only where identity/transport adaptation matters + +Typical tests: + +1. event -> projection -> command tests +2. recovery-choice tests +3. stale session / stale epoch rejection +4. stable `ReplicaID` versus mutable endpoint tests + +## Tier 1: Single-Volume Base Capability + +Goal: + +1. prove one volume is correct before adding replication + +Capabilities: + +1. create/delete +2. single-node read/write +3. restart durability +4. publication correctness +5. bounded observability + +Done means: + +1. RF=1 write/read survives restart +2. publication reflects the true serving node +3. explicit health/publication state is observable + +Primary proof tiers: + +1. core semantic for boundaries +2. integrated runtime for real read/write/restart + +Typical scenarios: + +1. create -> write -> restart -> read +2. publication remains coherent after restart + +## Tier 2: RF=2 Replication Base + +Goal: + +1. close the smallest useful HA replication unit + +Capabilities: + +1. primary/replica assignment +2. receiver readiness +3. shipper configuration +4. barrier semantics +5. explicit `publish_healthy` +6. explicit `degraded` +7. explicit `needs_rebuild` + +Done means: + +1. replica membership reaches the primary truthfully +2. `sync_all` cannot succeed vacuously with zero shippers +3. publication health depends on real closure, not optimistic state +4. RF=2 replicated write/read works on the integrated path + +Primary proof tiers: + +1. core semantic +2. seam +3. one integrated replicated IO scenario + +Typical tests: + +1. assignment-delivered membership tests +2. `RoleApplied`, `ReceiverReady`, `ShipperConfigured` closure tests +3. barrier strictness tests +4. replicated checksum scenarios + +## Tier 3: RF=2 Recovery And Failover + +Goal: + +1. turn RF=2 replication into a fault-tolerant runtime path + +Capabilities: + +1. manual promote +2. auto failover +3. old primary fencing +4. old primary rejoin +5. catch-up-first reconnect +6. rebuild fallback +7. data continuity after failover + +Done means: + +1. promotion bumps epoch and fences stale authority +2. promoted primary regains replica membership after rejoin +3. reconnect chooses catch-up or rebuild explicitly +4. failover preserves committed data +5. one data-verified integrated scenario exists for each supported failover path + +Primary proof tiers: + +1. seam +2. integrated runtime +3. soak/adversarial for disturbance variants + +Current note: + +1. manual promote on the integrated `weed` path has now closed with data continuity verification +2. this tier remains broader than one passing scenario and still requires systematic matrix expansion + +Typical scenarios: + +1. kill primary -> promote replica -> restart old primary -> data verified +2. lease-expiry auto failover +3. rejoin with address change +4. rebuild fallback when catch-up path is unavailable + +## Tier 4: Multi-Replica Runtime (`RF>=3`) + +Goal: + +1. extend the model from one replica to a replica set + +Capabilities: + +1. multi-replica membership +2. multi-shipper convergence +3. strict `sync_all` +4. `sync_quorum` +5. partial failure tolerance +6. replacement and rebuild target choice + +Done means: + +1. primary ownership and closure remain replica-scoped, not scalar-only +2. quorum/all durability rules hold under mixed replica states +3. failover and rejoin do not collapse back to RF=2-only assumptions + +Primary proof tiers: + +1. core semantic +2. seam +3. targeted integrated RF=3 scenarios + +Typical tests: + +1. multi-replica assignment closure +2. quorum durability tests +3. partial-failure promotion eligibility tests +4. RF=3 disturbance scenarios + +## Tier 5: Lifecycle Capability + +Goal: + +1. prove that product operations remain correct under replication and recovery + +Capabilities: + +1. expand +2. truncate +3. snapshot +4. snapshot export/import +5. clone/restore style flows where supported + +Done means: + +1. lifecycle operations preserve V2 recovery truth +2. lifecycle metadata does not bypass fencing or recovery boundaries +3. lifecycle operations continue to hold under restart/failover + +Primary proof tiers: + +1. core semantic for boundary rules +2. seam where command ownership matters +3. integrated scenarios for user-visible lifecycle behavior + +Typical scenarios: + +1. snapshot then failover +2. expand under replicated volume +3. truncate under degraded or catch-up conditions + +## Tier 6: Control Plane And Operations + +Goal: + +1. make the system diagnosable and operationally trustworthy + +Capabilities: + +1. heartbeat convergence +2. assignment queue correctness +3. registry truth coherence +4. publication truth coherence +5. debug surfaces +6. metrics and operator diagnosis +7. restart and disturbance policy clarity + +Done means: + +1. the control plane reports the same truth the runtime acts on +2. major failure classes are diagnosable from bounded logs/debug state +3. restart/rejoin behavior is policy-shaped, not accidental + +Primary proof tiers: + +1. seam +2. integrated runtime +3. soak for repeated disturbance + +Typical tests: + +1. registry/publication coherence tests +2. assignment queue confirm/refresh tests +3. reconnect/restart diagnosis tests +4. bounded failover observability tests + +## Tier 7: Product Surfaces (`CSI`, `iSCSI`, `NVMe`) + +Goal: + +1. project V2 storage truth through real product interfaces + +Capabilities: + +1. volume create/publish through `CSI` +2. node stage/node publish +3. failover-visible remount or reconnect behavior +4. expansion through product surface +5. snapshot through product surface +6. front-end publication coherence + +Done means: + +1. product surfaces do not hide or weaken V2 truth +2. frontend publication follows actual authority after failover +3. product workflows survive supported restart/failover envelopes + +Primary proof tiers: + +1. seam +2. integrated runtime +3. slower end-to-end scenario pack + +Typical scenarios: + +1. CSI create/publish/write/failover/read +2. CSI expand under replicated volume +3. snapshot + restore + failover + +## Tier 8: Launch Envelope + +Goal: + +1. convert bounded capability proof into a bounded support statement + +Capabilities: + +1. supported topology matrix +2. supported disturbance matrix +3. known unsupported branches +4. pilot stop conditions +5. rollout review evidence + +Done means: + +1. supported claims are explicit +2. unsupported areas are explicit +3. pilot and rollout review use the same capability map and proof layers + +Primary proof tiers: + +1. integrated runtime +2. soak / perf / operational review + +## Capability Map Summary + +| Tier | Scope | What closes here | Main proof emphasis | +|------|-------|------------------|---------------------| +| 0 | Semantic foundation | truth rules and fail-closed boundaries | core semantic | +| 1 | Single-volume base | RF=1 correctness and restart durability | core + integrated | +| 2 | RF=2 replication | receiver/shipper/barrier/publication closure | core + seam + one integrated path | +| 3 | RF=2 recovery/failover | promote, rejoin, catch-up, rebuild, data continuity | seam + integrated | +| 4 | RF>=3 runtime | multi-replica membership and durability semantics | core + seam + targeted integrated | +| 5 | Lifecycle | snapshot/expand/truncate under replication truth | mixed by feature | +| 6 | Control/ops | registry/heartbeat/publication/diagnosis closure | seam + integrated | +| 7 | Product surfaces | CSI and frontend projection of V2 truth | integrated | +| 8 | Launch envelope | bounded support and rollout claims | integrated + soak | + +## Test Expansion Strategy From This Map + +This map should drive testing in a faster order than "one expensive scenario at a time." + +### Fast lane + +Run on most code changes: + +1. core semantic tests for the touched rule +2. seam tests for ingress/egress/control delivery +3. one focused scenario only if the change crosses a real product seam + +### Medium lane + +Run on milestone closure for a tier: + +1. representative integrated scenarios for that tier +2. checksum or historical-read validation where data continuity matters + +### Slow lane + +Run on nightly or bounded review: + +1. disturbance matrix +2. soak +3. benchmark +4. larger product-surface packs + +## What Must Stay Runtime-Agnostic + +To avoid re-testing everything from zero when `weed` ownership shrinks later, +these proof categories must stay V2-owned: + +1. assignment semantics +2. role/epoch/fencing semantics +3. recovery-choice semantics +4. publication closure semantics +5. data continuity contracts + +The current `weed` path remains valuable as: + +1. the present integrated runtime +2. one proof backend for product-level behavior + +It must not become the only place where V2 truth is tested. + +## Immediate Next Use + +This map should be used to produce: + +1. one capability-to-test taxonomy +2. one current coverage matrix marking which tiers are: + - `strong` + - `bounded` + - `partial` + - `not yet closed` +3. one reduced high-value integrated scenario pack aligned to tiers rather than ad hoc bug history + +## Current Practical Reading + +For near-term work, read in this order: + +1. `v2-protocol-truths.md` +2. `v2-capability-map.md` +3. `v2-product-completion-overview.md` +4. `v2-phase-development-plan.md` +5. `v2_scenarios.md` diff --git a/sw-block/design/v2-kernel-closure-review.md b/sw-block/design/v2-kernel-closure-review.md new file mode 100644 index 000000000..700eee82b --- /dev/null +++ b/sw-block/design/v2-kernel-closure-review.md @@ -0,0 +1,143 @@ +# V2 Kernel Closure Review + +Date: 2026-04-05 +Status: active + +## Question + +The goal is not to prove whether iSCSI itself can be implemented. The reusable +`blockvol` + frontend code already shows that. + +The real question is whether the current kernel split can grow into a product: + +1. Is the brain owned by V2 semantics? +2. Is the control plane owned by V2 messages and convergence? +3. Is the data plane attached as an execution/backend service instead of a truth + owner? + +## Current Answer + +The current `masterv2 + volumev2 + purev2` shape is viable as a product kernel +because ownership is split in the right direction. + +### Brain + +Owner: + +- `sw-block/engine/replication/` + +What it owns: + +1. semantic state +2. event ingestion +3. command intent +4. outward projection + +What it must not own: + +1. backend I/O +2. transport lifecycle +3. frontend serving details + +### Control Plane + +Owner: + +- `sw-block/runtime/masterv2/` +- `sw-block/runtime/volumev2/control_session.go` +- `sw-block/runtime/volumev2/orchestrator.go` + +What it owns: + +1. desired declaration +2. heartbeat observation +3. assignment emission +4. assignment apply loop +5. convergence/idempotence + +What it must not own: + +1. WAL/extent execution +2. frontend protocol implementation + +### Data Plane + +Owner: + +- `sw-block/runtime/purev2/` +- `sw-block/runtime/volumev2/frontend.go` +- reused `weed/storage/blockvol/*` + +What it owns: + +1. create/open +2. read/write/flush +3. restart durability +4. frontend export such as iSCSI + +What it must not own: + +1. role truth +2. publication truth +3. assignment policy + +## Closure Proofs + +Two small closure proofs are enough for the current stage. + +### 1. Control-plane closure + +Scenario: + +1. `masterv2` declares one RF1 primary +2. `volumev2` heartbeats with no local role yet +3. `masterv2` emits an assignment +4. `volumev2` applies it through the V2 path +5. a later heartbeat converges to quiet state +6. if desired state changes, assignment is reissued once and converges again + +Why it matters: + +- this proves the new head is not piggybacking on `weed/server` loops + +### 2. Data-plane closure + +Scenario: + +1. `volumev2` exports a named volume through iSCSI +2. a client logs in and issues SCSI write/read +3. data is verified through the frontend and local backend view + +Why it matters: + +- this proves the kernel can host a real frontend while keeping truth ownership + outside the frontend/backend code + +## Product Meaning + +If these two closures stay true while features expand, then the architecture can +scale toward: + +1. RF1 productized single-node block service +2. RF2/RF3 replication as additional control/data workflows +3. failover and rebuild without moving semantic truth back into backend code +4. CSI on top of a clearer runtime contract + +Another way to state the same result: + +1. `masterv2` behaves like an external identity authority +2. each `volumev2` instance behaves like a per-volume micro-cluster shell +3. the selected primary inside that shell owns data-control truth and recovery + choreography + +## Main Risk + +The main risk is not iSCSI or local I/O. The main risk is semantic leakage: + +1. adding more backend-state shortcuts into control decisions +2. letting frontend/backend code redefine publication truth +3. rebuilding `weed/server` ownership inside `volumev2` +4. letting `masterv2` grow from identity authority into a centralized recovery + planner + +As long as those three are resisted, the kernel can keep expanding cleanly. diff --git a/sw-block/design/v2-loop1-surface-draft.md b/sw-block/design/v2-loop1-surface-draft.md new file mode 100644 index 000000000..07c1094aa --- /dev/null +++ b/sw-block/design/v2-loop1-surface-draft.md @@ -0,0 +1,253 @@ +# V2 Loop1 Surface Draft + +Date: 2026-04-05 +Status: active + +## Purpose + +This note turns the two-loop design into a code-facing draft for the current +`masterv2` and `volumev2` packages. + +It does not implement the new protocol yet. +It defines the smallest surface refactor that should happen first. + +## Goal + +Replace the current mixed `heartbeat -> assignments` MVP surface with three +separate `Loop 1` surfaces: + +1. periodic heartbeat +2. promotion query +3. assignment + +## Current State + +Today, `masterv2` uses one heartbeat type and one assignment type: + +- `sw-block/runtime/masterv2/master.go` +- `sw-block/runtime/volumev2/control_session.go` +- `sw-block/runtime/volumev2/volume.go` + +The current heartbeat still mixes: + +1. applied identity +2. outward projection naming +3. fields that should later become failover evidence + +## Target Surfaces + +### 1. Heartbeat + +Purpose: + +1. liveness +2. applied identity +3. outward compressed mode + +Recommended fields: + +- `NodeID` +- `ReportedAt` +- per-volume: + - `Name` + - `Path` + - `Epoch` + - `Role` + - `Mode` + - `ModeReason` + - passive `CommittedLSN` cache + - `RoleApplied` + - `ReplicaReady` + +Not included: + +- per-replica progress +- catch-up target +- detailed recovery phase + +If `CommittedLSN` is present here, it is only a convenience cache for +`masterv2`. Fresh promotion judgment still uses the query channel. + +### 2. Promotion Query + +Purpose: + +1. fresh promotion evidence +2. failover arbitration + +Recommended request: + +- `VolumeName` +- `ExpectedEpoch` + +Recommended response: + +- `VolumeName` +- `NodeID` +- `Epoch` +- `Role` +- `CommittedLSN` +- `WALHeadLSN` +- `ReceiverReady` +- `Eligible` +- `Reason` + +Rule: + +- `CommittedLSN` is the primary selection key +- `WALHeadLSN` is only a tiebreaker + +### 3. Assignment + +Purpose: + +1. authorize role +2. fence stale owners +3. deliver member identity + +Recommended fields: + +- `Name` +- `Path` +- `NodeID` +- `Epoch` +- `LeaseTTL` +- `Role` +- `ReplicaSet` +- `CreateOptions` + +`ReplicaSet` should carry identity plus transport addresses, not progress. + +## Field Migration From Current MVP + +### Current `masterv2.VolumeHeartbeat` + +Today: + +- `Name` +- `Path` +- `Epoch` +- `Role` +- `ProjectionMode` +- `PublicationReason` +- `RoleApplied` + +Should become: + +- `Name` +- `Path` +- `Epoch` +- `Role` +- `Mode` +- `ModeReason` +- `RoleApplied` +- `ReplicaReady` + +Interpretation change: + +- `ProjectionMode` should be renamed to `Mode` +- `PublicationReason` should stop pretending to be a generic reason field +- `ReplicaReady` should be explicit on the identity surface + +### Current `masterv2.VolumeView` + +Today: + +- `ObservedEpoch` +- `ObservedRole` +- `ProjectionMode` +- `PublicationReason` +- `RoleApplied` + +Should become: + +- `ObservedEpoch` +- `ObservedRole` +- `Mode` +- `ModeReason` +- `RoleApplied` +- `ReplicaReady` + +Optional later: + +- bounded cached failover evidence for debugging only + +### Current `volumev2.Node.Heartbeat()` + +Current source: + +- `snap.Status` +- `snap.Projection` + +Recommended extraction: + +- `Mode` from `snap.Projection.Mode.Name` +- `ModeReason` from `snap.Projection.Mode.Reason` +- passive `CommittedLSN` cache from local status snapshot +- `RoleApplied` from `snap.Projection.Readiness.RoleApplied` +- `ReplicaReady` from `snap.Projection.Readiness.ReplicaReady` + +Not from heartbeat: + +- `CatchUpTarget` +- `RecoveryProgress` + +`CommittedLSN` may appear in heartbeat as a passive cache only. +Fresh promotion authority still belongs to the promotion-query channel. +`CatchUpTarget` and `RecoveryProgress` belong to Loop 2. + +## Code Refactor Order + +### Step 1 + +Add a small shared contract package for `Loop 1` types, for example under: + +- `sw-block/runtime/protocolv2/` + +Start with: + +1. `Heartbeat` +2. `Assignment` +3. `PromotionQueryRequest` +4. `PromotionQueryResponse` + +### Step 2 + +Make `masterv2` use the shared `Loop 1` contract types instead of local ad hoc +duplicates. + +### Step 3 + +Make `volumev2` heartbeat generation write to the narrowed heartbeat shape. + +### Step 4 + +Add a promotion-query interface without implementing full failover yet. + +The first code slice only needs: + +1. request and response types +2. local evidence extraction helper +3. one focused test proving query returns fresh local evidence + +## Test Guidance + +The first code refactor should keep existing control-loop tests and add one new +test: + +1. existing heartbeat-assignment convergence should still pass +2. new promotion-query test should prove: + - query is separate from heartbeat + - fresh state is returned at call time + - heartbeat `CommittedLSN` is only a passive cache + - `WALHeadLSN` is not part of periodic heartbeat + +## Non-Goals + +This draft does not yet define: + +1. full Loop 2 message schema +2. full new primary truth reconstruction choreography +3. quorum-specific `CommittedLSN` algorithm + +Those come after the `Loop 1` surface is cleanly split. diff --git a/sw-block/design/v2-proof-and-retest-pyramid.md b/sw-block/design/v2-proof-and-retest-pyramid.md new file mode 100644 index 000000000..918634f7b --- /dev/null +++ b/sw-block/design/v2-proof-and-retest-pyramid.md @@ -0,0 +1,186 @@ +# V2 Proof And Retest Pyramid + +Date: 2026-04-05 +Status: active + +## Purpose + +This note defines how the pure V2 runtime should accumulate proof so that most +closure stays in fast, reusable tests instead of expensive mixed scenarios. + +It also defines where reuse is allowed to keep narrow regression coverage and +where truth-owner changes require full V2 retesting. + +## Proof Pyramid + +### Layer 1: core semantic tests + +Owner: + +- `sw-block/engine/replication/` + +This layer proves: + +1. assignment semantics +2. role-application semantics +3. readiness/publication semantics +4. stale/replay/idempotence behavior +5. recovery and fencing meaning + +Rule: + +- if a behavior changes V2 protocol truth, this layer is mandatory + +### Layer 2: component and seam tests + +Owner: + +- `sw-block/runtime/purev2/` +- narrow bridge/dispatcher seams reused from `weed/` + +This layer proves: + +1. static assignment ingestion +2. dispatcher to backend binding +3. local role application feedback into the core +4. local restart/open/replay behavior +5. debug/projection cache consistency + +Rule: + +- this is the default daily development surface for the new runtime + +### Layer 3: minimal integrated runtime tests + +Owner: + +- pure runtime binary/package smoke + +This layer proves only: + +1. create -> assign -> write -> read +2. restart -> reopen -> read +3. status/debug snapshot visibility + +Rule: + +- keep this pack tiny and fast +- do not pull failover, multi-node, or CSI into this layer + +### Layer 4: compatibility and oracle checks + +Owner: + +- current `weed` integrated path + +This layer remains useful for: + +1. regression oracle coverage +2. parity comparison +3. late acceptance confidence + +Rule: + +- it is not the daily semantic development surface for pure V2 + +## Reuse Versus Full Retest + +### Reuse with narrow regression + +The following areas remain execution muscles and should keep focused coverage: + +1. WAL append and flush mechanics +2. checkpoint and dirty-map mechanics +3. extent install mechanics +4. local read/write I/O +5. transport/frontend plumbing when semantic meaning is unchanged + +Typical proof: + +1. component tests +2. mechanical regression tests +3. one narrow smoke if the seam changed + +### Full V2 retest required + +The following areas change truth ownership and therefore require explicit V2 proof: + +1. assignment meaning +2. publication meaning +3. readiness closure +4. fencing and lineage meaning +5. recovery classification +6. operator-visible health and degraded semantics + +Typical proof: + +1. core semantic tests first +2. pure-runtime component tests second +3. only then one small integrated confirmation + +## Stage Gates + +### Stage A: RF1 pure shell + +Must be green before any RF2 work starts: + +1. create/open works +2. static primary assignment works +3. local write/read works +4. restart durability works +5. debug/projection snapshot works + +### Stage B: RF2 replication base + +May start only after Stage A is closed. + +This stage adds: + +1. replica membership truth +2. receiver/shipper wiring +3. barrier semantics +4. degraded versus publish closure +5. catch-up and rebuild base semantics + +### Stage C: failover and rejoin + +May start only after RF2 replication base is closed. + +This stage adds: + +1. manual promote +2. auto failover +3. rejoin +4. recovery ownership handoff + +### Stage D: product surfaces + +May start only after failover/rejoin closure exists. + +This stage adds: + +1. CSI +2. operator APIs +3. external readiness and health surfaces +4. acceptance and soak packs + +## Daily Working Rule + +When a new bug appears, classify it first: + +1. core semantic bug +2. pure-runtime seam bug +3. execution-muscle bug +4. compatibility-only oracle bug + +Then choose the cheapest proof tier that still matches the truth owner. + +If the answer is "mixed scenario first", the classification is probably still +too vague. + +## Related References + +- `v2-pure-runtime-rf1-bootstrap.md` +- `v2-capability-map.md` +- `v2-reuse-replacement-boundary.md` +- `v2-legacy-runtime-exit-criteria.md` diff --git a/sw-block/design/v2-pure-runtime-rf1-bootstrap.md b/sw-block/design/v2-pure-runtime-rf1-bootstrap.md new file mode 100644 index 000000000..1fac087c5 --- /dev/null +++ b/sw-block/design/v2-pure-runtime-rf1-bootstrap.md @@ -0,0 +1,148 @@ +# Pure V2 RF1 Bootstrap + +Date: 2026-04-05 +Status: active + +## Purpose + +This note turns the bootstrap plan into a concrete runtime boundary that can be +implemented and tested without going through `weed/server`. + +The goal of the first executable slice is not feature completeness. + +The goal is to establish one small, closed, V2-owned runtime that can accumulate +semantic truth and local execution proof without mixed-runtime distortion. + +## Implemented Boundary + +### Pure V2 shell + +The first pure runtime shell lives in: + +- `sw-block/runtime/purev2/` + +It owns: + +- local process/runtime lifecycle +- local block volume registration +- V2 core engine ownership +- local projection/debug cache +- static RF1 assignment injection + +It does not own: + +- master heartbeat loops +- assignment queues +- failover control +- recovery task orchestration +- CSI or product-facing control APIs + +### Reused execution muscles + +The pure shell reuses mechanics instead of re-implementing them: + +- `weed/storage/store_blockvol.go` +- `weed/storage/blockvol/blockvol.go` +- `weed/storage/blockvol/v2bridge/command_bindings.go` +- `weed/server/blockcmd/` + +The intended rule is: + +- `sw-block/runtime/purev2` owns local runtime closure +- `sw-block/engine/replication` owns semantic authority +- reused `weed/` pieces execute concrete backend actions only + +### Small executable entrypoint + +The first operator-facing entrypoint lives in: + +- `sw-block/cmd/purev2rf1/` + +Current commands: + +1. `bootstrap` +2. `status` + +This is intentionally small. + +It exists to make the first slice executable and inspectable, not to define the +final product surface. + +## RF1 First Slice Contract + +The first slice is closed only if all of the following are true. + +### Included + +1. create one local block volume +2. open an existing local block volume +3. inject one static RF1 primary assignment +4. apply local role through V2 command dispatch +5. perform real local read/write through `blockvol` +6. survive restart and preserve data +7. expose explicit projection/debug state + +### Explicitly excluded + +1. replica membership truth +2. receiver/shipper wiring as a required path +3. catch-up or rebuild orchestration +4. manual or auto failover +5. CSI + +If a new requirement needs any excluded item, it belongs to a later tier and +must not be forced into the RF1 shell. + +## Runtime Shape + +The implemented ownership split is: + +```text +purev2 runtime +-> V2 core engine +-> dispatcher +-> command bindings +-> blockvol store/backend +-> projection/debug snapshot +``` + +The critical closure path is: + +```text +static assignment +-> core ApplyEvent(AssignmentDelivered) +-> dispatcher executes apply_role +-> runtime feeds back RoleApplied +-> projection is cached explicitly +-> local debug snapshot becomes inspectable +``` + +## Current Behavioral Meaning + +For the current engine semantics, an RF1 primary with zero replicas remains: + +1. locally writable as a block volume +2. explicitly visible in debug/projection state +3. projected as `allocated_only`, not `publish_healthy` + +That is acceptable for this first bootstrap slice because: + +1. the purpose is shell closure, not final RF1 publication semantics +2. publication meaning remains explicit instead of being guessed from local role +3. RF1 publication policy can evolve later without changing the shell boundary + +## Immediate Engineering Rules + +While the pure runtime remains in the RF1 stage: + +1. new shell work goes into `sw-block/runtime/purev2/` +2. pure runtime must not import `weed/server/volume_server_block.go` +3. new proof should prefer unit/component tests in the pure shell +4. mixed `weed` scenario results remain oracle coverage, not semantic authority + +## Related References + +- `v2-capability-map.md` +- `v2-reuse-replacement-boundary.md` +- `v2-legacy-runtime-exit-criteria.md` +- `v2-protocol-truths.md` diff --git a/sw-block/design/v2-two-loop-protocol.md b/sw-block/design/v2-two-loop-protocol.md new file mode 100644 index 000000000..121f3d71e --- /dev/null +++ b/sw-block/design/v2-two-loop-protocol.md @@ -0,0 +1,414 @@ +# V2 Two-Loop Protocol + +Date: 2026-04-05 +Status: active + +## Purpose + +This note fixes the protocol boundary for the next V2 step. + +The goal is not to finalize every wire field before implementation. +The goal is to make the ownership boundary stable enough that automata, +constraints, and runtime packages can be reorganized without mixing identity +control and replication consensus again. + +## Core Rule + +The protocol is split into two loops: + +1. `Loop 1`: identity control +2. `Loop 2`: data control + +These loops must not be collapsed into one heartbeat or one state owner. + +## Authority Principle + +Each volume should be treated as a small distributed cluster: + +1. `masterv2` is the identity authority outside the cluster +2. the selected primary is the data-control authority inside the cluster +3. replicas report bounded facts to the primary, not full truth to `masterv2` + +This means: + +1. `masterv2` decides who is allowed to own the role +2. the new primary decides how takeover, catch-up, and rebuild proceed +3. `masterv2` may query bounded facts for arbitration, but it does not choreograph + data recovery step by step + +## Loop 1: Identity Control + +Owner: + +- `masterv2 <-> volumev2` + +Frequency: + +- low +- heartbeat scale +- assignment scale +- promotion-query scale + +## Three Control Channels + +Within `Loop 1`, the control plane should be split into three different +channels. They must not be collapsed into one message type. + +### 1. Heartbeat + +Direction: + +- `volumev2 -> masterv2` + +Frequency: + +- periodic +- lightweight + +Purpose: + +1. liveness detection +2. compressed outward mode +3. confirmation that assignment was applied + +Heartbeat should carry only: + +1. `NodeID` +2. per-volume `Mode` +3. applied `Epoch` +4. applied `Role` +5. `RoleApplied` +6. `ReplicaReady` +7. optional passive `CommittedLSN` cache + +Heartbeat should not carry: + +1. per-replica progress +2. catch-up targets +3. rebuild detail +4. full failover evidence + +If `CommittedLSN` is carried in heartbeat, it is only a passive cache. The +authoritative failover-time value still comes from promotion query. + +### 2. Promotion Query + +Direction: + +- `masterv2 -> candidate volumev2` +- candidate `volumev2 -> masterv2` + +Frequency: + +- on demand +- only during failover or promotion arbitration + +Purpose: + +1. obtain fresh failover evidence +2. avoid treating stale heartbeat cache as authority + +Candidate response should include: + +1. `CommittedLSN` +2. `WALHeadLSN` as a weaker tiebreaker +3. `Epoch` +4. `Role` +5. `ReceiverReady` +6. bounded eligibility reason if not promotable + +The promotion query is where fresh identity-loop evidence is collected. +It is not a replacement for the data-control loop, and it is not a continuous +replication-progress feed. + +If heartbeat also carries `CommittedLSN`, promotion query still wins whenever +fresh arbitration is required. + +### 3. Assignment + +Direction: + +- `masterv2 -> volumev2` + +Frequency: + +- on demand +- role change or membership change + +Purpose: + +1. authorize role ownership +2. fence stale owners +3. deliver replica-set identity + +### Master To Volume + +`masterv2 -> volumev2` carries only: + +1. `Epoch` +2. `Role` +3. `LeaseTTL` +4. `ReplicaSet` identities and addresses + +It does not carry: + +1. per-replica progress +2. catch-up target history +3. detailed rebuild plan + +### Volume To Master + +`volumev2 -> masterv2` carries only bounded identity evidence: + +1. applied `Epoch` +2. applied `Role` +3. outward `Mode` +4. `RoleApplied` +5. `ReplicaReady` + +This channel must remain lightweight. Fresh failover evidence belongs to the +promotion-query channel, not the periodic heartbeat. + +## Loop 2: Data Control + +Owner: + +- `primary engine <-> replica engine` + +Frequency: + +- high +- write scale +- barrier scale +- reconnect scale + +This is where replication consensus lives. + +### Primary To Replica + +Primary-side data-control messages should cover: + +1. WAL entry stream +2. barrier request with `Epoch` and target LSN +3. reconnect or resume handshake +4. rebuild/catch-up execution requests when needed + +### Replica To Primary + +Replica-side data-control messages should cover: + +1. `FlushedLSN` +2. bounded status such as `ok`, `epoch_mismatch`, `timeout`, `fsync_failed` +3. reconnect gap evidence +4. coarse local recovery state + +### Primary-Owned Per-Replica State + +The primary brain should own: + +1. `ReplicaFlushedLSN` +2. `ShippedLSN` as diagnostic only +3. replica `State` +4. `CatchUpTarget` +5. `RetentionFloor` +6. `LastContactTime` + +## Role Of Masterv2 + +`masterv2` authorizes: + +1. who is primary +2. who is replica +3. which epoch is active +4. when stale owners must be fenced + +`masterv2` must not decide: + +1. replay from LSN `X` to `Y` +2. whether the next action is `keepup` or `catchup` +3. how rebuild is executed +4. continuous commit progress + +`masterv2` may query candidates for fresh promotion evidence, but it still does +not become the owner of replication history. + +## Reconstruction And Takeover + +Promotion and reconstruction are related, but they do not have the same owner. + +### What Masterv2 Leads + +`masterv2` leads: + +1. failover detection +2. epoch fencing +3. candidate query for fresh promotion evidence +4. primary selection +5. assignment of the new primary role + +### What The New Primary Leads + +The selected replacement primary leads: + +1. local assignment realization +2. collection of self and peer replica summaries +3. bounded truth reconstruction +4. fail-closed takeover gating +5. follow-on `keepup`, `catchup`, or `rebuild` orchestration + +### Rule + +`masterv2` may say: + +1. "you are now the authorized primary candidate for epoch `E`" +2. "these are the members of the replica set" + +But `masterv2` must not say: + +1. "replay from LSN `X` to `Y`" +2. "use replica `R` as the rebuild source" +3. "enter `catchup` before `rebuild`" +4. "the cluster is safe because my last cached view looked healthy" + +## Role Of Primary Brain + +The primary brain discovers: + +1. current replica state +2. gap or retention situation +3. barrier success or failure +4. whether the volume is `keepup`, `catchup`, `degraded`, or `needs_rebuild` + +The primary brain decides: + +1. keep shipping +2. start catch-up +3. escalate to `needs_rebuild` +4. start rebuild after role/assignment allows it + +## Distributed State-Machine Rules + +### 1. Different Nodes Have Different Views + +Each node must distinguish: + +1. local execution truth +2. last observed peer truth +3. cluster identity truth from `masterv2` + +Do not collapse these into one blob. + +### 2. All Peer Observations Are Epoch-Scoped + +Any peer observation that affects recovery must be tied to: + +1. `Epoch` +2. session or generation token + +Old-epoch observations must be ignored or fail closed. + +### 3. New Primary Reconstructs Truth + +After failover, the new primary must rebuild its own data-control truth from: + +1. local state +2. peer summaries +3. reconnect handshakes + +It must not trust `masterv2` as a cache of full recovery history. +It may use `masterv2` only as the source of authorization and replica identity. + +### 4. Outward Mode Is Compressed Evidence + +`allocated_only`, `bootstrap_pending`, `publish_healthy`, `degraded`, and +`needs_rebuild` are public meanings, not the full internal recovery automaton. + +### 5. Ambiguity Fails Closed + +When barrier lineage, progress lineage, or epoch lineage is unclear, the system +must prefer: + +1. `degraded` +2. `needs_rebuild` +3. no promotion without enough eligibility evidence + +## Constraint Migration + +Most of the last week's V2 work remains valid. The important change is where +each constraint belongs. + +### Keep As-Is + +These constraints still stand: + +1. epoch fencing +2. one active session per replica per epoch +3. `catchup` and `rebuild` are different paths +4. fail closed on ambiguous recovery truth +5. semantics first, adapters later + +### Move To Loop 1 + +These belong to identity control: + +1. assignment application +2. role ownership +3. lease ownership +4. stable replica identity and addressing +5. compressed heartbeat evidence +6. on-demand promotion query for fresh evidence + +### Move To Loop 2 + +These belong to data control: + +1. committed/durable/checkpoint boundaries +2. barrier result meaning +3. replica progress +4. keepup/catchup/rebuild progression +5. retention-floor and catch-up targeting + +## Existing V2 Seeds To Reuse + +The current V2 code already has the right seeds for the primary-led loop: + +1. `sw-block/engine/replication/state.go` +2. `sw-block/engine/replication/event.go` +3. `sw-block/engine/replication/command.go` +4. `sw-block/engine/replication/sender.go` +5. `sw-block/engine/replication/session.go` +6. `sw-block/engine/replication/registry.go` + +The current MVP already has the right seeds for the identity loop: + +1. `sw-block/runtime/masterv2/master.go` +2. `sw-block/runtime/volumev2/control_session.go` +3. `sw-block/runtime/volumev2/orchestrator.go` + +## Immediate Next Step + +Before deeper implementation, the codebase should next define: + +1. the minimal `Loop 1` contract types in code, split into heartbeat, promotion + query, and assignment +2. the minimal `Loop 2` progress and reconnect contract draft +3. the automata ownership map showing which engine events and commands belong + to identity control versus data control + +## Promotion Logic + +Promotion should use fresh on-demand evidence, not stale heartbeat cache. + +Recommended judgment order: + +1. fence the old primary by epoch +2. query all surviving candidates +3. reject any candidate with wrong epoch, wrong role lineage, or not-ready + receiver state +4. choose the candidate with highest `CommittedLSN` +5. use `WALHeadLSN` only as a tiebreaker for equally committed candidates +6. assign new primary role at a new epoch + +This keeps the durability boundary centered on `CommittedLSN`, which is the +last LSN that satisfied the configured durability mode such as `sync_all` or +`sync_quorum`. diff --git a/sw-block/design/v2-volumev2-single-node-mvp.md b/sw-block/design/v2-volumev2-single-node-mvp.md new file mode 100644 index 000000000..fafec8307 --- /dev/null +++ b/sw-block/design/v2-volumev2-single-node-mvp.md @@ -0,0 +1,151 @@ +# V2 VolumeV2 Single-Node MVP + +Date: 2026-04-05 +Status: active + +## Purpose + +This note defines the target shape for a single-node `volumev2` MVP that can +ship as a normal block service before HA/failover exists. + +The core idea is: + +1. `masterv2` is fully new control ownership +2. `volumev2` is a new shell and brain host +3. `blockvol` and related backend mechanics remain reusable muscles + +## Target Layering + +`volumev2` should be strengthened around four layers. + +### 1. Engine + +Owner: + +- `sw-block/engine/replication/` + +Responsibility: + +1. state +2. event ingestion +3. command emission +4. outward projection + +Rule: + +- semantic truth lives here +- no backend I/O or network ownership + +### 2. Engine Interface + +Owner: + +- command/event vocabulary between control/runtime and backend execution + +Responsibility: + +1. assignment -> event translation +2. observation -> event translation +3. command -> execution dispatch contract + +Rule: + +- runtime shell may not mutate engine truth directly + +### 3. Control Plane + +Owner: + +- `masterv2 <-> volumev2` coordination + +Responsibility: + +1. node identity +2. registration and heartbeat +3. assignment receipt +4. state reporting +5. future recovery-control vocabulary (`keepup`, `catchup`, `rebuild`) + +Rule: + +- control plane carries protocol messages +- it does not own local data execution + +### 4. Data Plane + +Owner: + +- local storage and serving mechanics + +Responsibility: + +1. WAL/extent management +2. read/write/flush +3. background workers +4. receiver/shipper mechanics +5. NVMe/iSCSI/frontend serving + +Rule: + +- data plane knows how to execute +- it does not define publication or role semantics + +## Single-Node MVP Contract + +The first ship-capable `volumev2` slice should include: + +1. `masterv2` declaration of one RF1 primary volume +2. `volumev2` control session to fetch assignments +3. local create/open through reused `blockvol` +4. local primary assignment application through the V2 engine +5. local read/write plus restart durability +6. debug/status snapshot +7. one small executable entrypoint for smoke usage + +The first slice explicitly excludes: + +1. failover +2. RF2 replication +3. catch-up/rebuild ownership +4. CSI + +## Why This Is Enough + +This is enough to prove: + +1. the `masterv2 + volumev2` head is viable +2. `volumev2` can host V2 semantics while reusing V1 muscles +3. a useful non-HA block service can exist before HA complexity is added + +## Module Shape + +Recommended package split: + +1. `sw-block/runtime/masterv2/` +2. `sw-block/runtime/volumev2/` +3. `sw-block/runtime/purev2/` +4. `sw-block/engine/replication/` +5. `sw-block/bridge/blockvol/` + +Within `volumev2`, strengthen toward: + +1. `control_session.go` +2. `orchestrator.go` +3. `node.go` +4. later: `heartbeat.go`, `frontend.go`, `workers.go` + +## Stage Gate + +`volumev2` may be treated as a single-node MVP only when: + +1. assignment sync is repeatable and idempotent +2. local IO is data-verified +3. restart/open path is proven +4. status/debug state is explicit +5. no `weed/server` lifecycle owner is required + +## Related References + +- `v2-pure-runtime-rf1-bootstrap.md` +- `v2-proof-and-retest-pyramid.md` +- `v2-capability-map.md` diff --git a/sw-block/runtime/masterv2/master.go b/sw-block/runtime/masterv2/master.go new file mode 100644 index 000000000..d41693b93 --- /dev/null +++ b/sw-block/runtime/masterv2/master.go @@ -0,0 +1,264 @@ +package masterv2 + +import ( + "fmt" + "slices" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// Config defines the minimal masterv2 control-loop settings. +type Config struct { + LeaseTTL time.Duration +} + +// VolumeSpec is one master-owned desired volume intent. +type VolumeSpec struct { + Name string + Path string + PrimaryNodeID string + CreateOptions blockvol.CreateOptions +} + +// Assignment is the minimal masterv2 -> volumev2 control message. +type Assignment = protocolv2.Assignment + +// VolumeHeartbeat is the minimal volumev2 -> masterv2 periodic identity observation. +type VolumeHeartbeat = protocolv2.VolumeHeartbeat + +// NodeHeartbeat is the minimal per-node heartbeat used by the POC control loop. +type NodeHeartbeat = protocolv2.NodeHeartbeat + +// PromotionQueryRequest asks one candidate for fresh failover evidence. +type PromotionQueryRequest = protocolv2.PromotionQueryRequest + +// PromotionQueryResponse returns fresh candidate evidence at query time. +type PromotionQueryResponse = protocolv2.PromotionQueryResponse + +// VolumeView is the master-side observed state for one desired volume. +type VolumeView struct { + Name string + Path string + PrimaryNodeID string + DesiredEpoch uint64 + ObservedEpoch uint64 + ObservedRole string + Mode string + ModeReason string + CommittedLSN uint64 + RoleApplied bool + ReplicaReady bool + LastHeartbeatAt time.Time +} + +type desiredVolume struct { + spec VolumeSpec + epoch uint64 +} + +// Master is a small in-process V2 control plane POC. +// It owns desired state and emits assignments on heartbeats. +type Master struct { + mu sync.RWMutex + cfg Config + desired map[string]desiredVolume + views map[string]VolumeView +} + +// New creates the minimal masterv2 POC. +func New(cfg Config) *Master { + if cfg.LeaseTTL <= 0 { + cfg.LeaseTTL = 30 * time.Second + } + return &Master{ + cfg: cfg, + desired: make(map[string]desiredVolume), + views: make(map[string]VolumeView), + } +} + +// DeclarePrimary records one desired RF1 primary placement. +func (m *Master) DeclarePrimary(spec VolumeSpec) error { + if spec.Name == "" { + return fmt.Errorf("masterv2: volume name is required") + } + if spec.Path == "" { + return fmt.Errorf("masterv2: volume path is required") + } + if spec.PrimaryNodeID == "" { + return fmt.Errorf("masterv2: primary node id is required") + } + + m.mu.Lock() + defer m.mu.Unlock() + + entry, ok := m.desired[spec.Name] + if !ok { + entry = desiredVolume{epoch: 1} + } else if entry.spec.Path != spec.Path || entry.spec.PrimaryNodeID != spec.PrimaryNodeID || entry.spec.CreateOptions != spec.CreateOptions { + entry.epoch++ + } + entry.spec = spec + m.desired[spec.Name] = entry + view := m.views[spec.Name] + view.Name = spec.Name + view.Path = spec.Path + view.PrimaryNodeID = spec.PrimaryNodeID + view.DesiredEpoch = entry.epoch + m.views[spec.Name] = view + return nil +} + +// HandleHeartbeat consumes one node heartbeat and returns any assignments the node should apply. +func (m *Master) HandleHeartbeat(hb NodeHeartbeat) ([]Assignment, error) { + if hb.NodeID == "" { + return nil, fmt.Errorf("masterv2: heartbeat node id is required") + } + if hb.ReportedAt.IsZero() { + hb.ReportedAt = time.Now() + } + + m.mu.Lock() + defer m.mu.Unlock() + + byName := make(map[string]VolumeHeartbeat, len(hb.Volumes)) + for _, vol := range hb.Volumes { + byName[vol.Name] = vol + view := m.views[vol.Name] + view.Name = vol.Name + view.Path = vol.Path + view.ObservedEpoch = vol.Epoch + view.ObservedRole = vol.Role + view.Mode = vol.Mode + view.ModeReason = vol.ModeReason + view.CommittedLSN = vol.CommittedLSN + view.RoleApplied = vol.RoleApplied + view.ReplicaReady = vol.ReplicaReady + view.LastHeartbeatAt = hb.ReportedAt + if desired, ok := m.desired[vol.Name]; ok { + view.PrimaryNodeID = desired.spec.PrimaryNodeID + view.DesiredEpoch = desired.epoch + } + m.views[vol.Name] = view + } + + var assignments []Assignment + names := make([]string, 0, len(m.desired)) + for name := range m.desired { + names = append(names, name) + } + slices.Sort(names) + + for _, name := range names { + desired := m.desired[name] + if desired.spec.PrimaryNodeID != hb.NodeID { + continue + } + reported, ok := byName[name] + if ok && reported.Path == desired.spec.Path && reported.Epoch == desired.epoch && reported.Role == "primary" && reported.RoleApplied { + continue + } + assignments = append(assignments, m.primaryAssignment(desired)) + } + return assignments, nil +} + +// Volume returns the latest master-side view for one volume. +func (m *Master) Volume(name string) (VolumeView, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + view, ok := m.views[name] + return view, ok +} + +// SelectPromotionCandidate chooses the best eligible candidate from fresh +// promotion-query responses. Selection is durability-first: highest +// CommittedLSN wins, then WALHeadLSN as a weaker tiebreaker. +func (m *Master) SelectPromotionCandidate(responses []PromotionQueryResponse) (PromotionQueryResponse, error) { + candidates := make([]PromotionQueryResponse, 0, len(responses)) + for _, resp := range responses { + if resp.Eligible { + candidates = append(candidates, resp) + } + } + if len(candidates) == 0 { + return PromotionQueryResponse{}, fmt.Errorf("masterv2: no eligible promotion candidates") + } + slices.SortStableFunc(candidates, func(a, b PromotionQueryResponse) int { + if a.CommittedLSN != b.CommittedLSN { + if a.CommittedLSN > b.CommittedLSN { + return -1 + } + return 1 + } + if a.WALHeadLSN != b.WALHeadLSN { + if a.WALHeadLSN > b.WALHeadLSN { + return -1 + } + return 1 + } + switch { + case a.NodeID < b.NodeID: + return -1 + case a.NodeID > b.NodeID: + return 1 + default: + return 0 + } + }) + return candidates[0], nil +} + +// AuthorizePromotion selects the best candidate from fresh promotion evidence, +// advances desired ownership when needed, and returns the assignment the chosen +// node should apply. This is authorization only; takeover reconstruction and +// activation remain the new primary's responsibility. +func (m *Master) AuthorizePromotion(volumeName string, responses []PromotionQueryResponse) (Assignment, error) { + if volumeName == "" { + return Assignment{}, fmt.Errorf("masterv2: volume name is required") + } + selected, err := m.SelectPromotionCandidate(responses) + if err != nil { + return Assignment{}, err + } + if selected.VolumeName != "" && selected.VolumeName != volumeName { + return Assignment{}, fmt.Errorf("masterv2: promotion candidate volume %q does not match %q", selected.VolumeName, volumeName) + } + + m.mu.Lock() + defer m.mu.Unlock() + + desired, ok := m.desired[volumeName] + if !ok { + return Assignment{}, fmt.Errorf("masterv2: unknown volume %q", volumeName) + } + if desired.spec.PrimaryNodeID != selected.NodeID { + desired.spec.PrimaryNodeID = selected.NodeID + desired.epoch++ + m.desired[volumeName] = desired + } + + view := m.views[volumeName] + view.Name = desired.spec.Name + view.Path = desired.spec.Path + view.PrimaryNodeID = desired.spec.PrimaryNodeID + view.DesiredEpoch = desired.epoch + m.views[volumeName] = view + + return m.primaryAssignment(desired), nil +} + +func (m *Master) primaryAssignment(desired desiredVolume) Assignment { + return Assignment{ + Name: desired.spec.Name, + Path: desired.spec.Path, + NodeID: desired.spec.PrimaryNodeID, + Epoch: desired.epoch, + LeaseTTL: m.cfg.LeaseTTL, + CreateOptions: desired.spec.CreateOptions, + Role: "primary", + } +} diff --git a/sw-block/runtime/masterv2/master_test.go b/sw-block/runtime/masterv2/master_test.go new file mode 100644 index 000000000..bd9e4d469 --- /dev/null +++ b/sw-block/runtime/masterv2/master_test.go @@ -0,0 +1,138 @@ +package masterv2 + +import "testing" + +func TestSelectPromotionCandidate_PrefersHighestCommittedLSN(t *testing.T) { + master := New(Config{}) + selected, err := master.SelectPromotionCandidate([]PromotionQueryResponse{ + { + VolumeName: "vol-a", + NodeID: "node-b", + CommittedLSN: 12, + WALHeadLSN: 20, + Eligible: true, + }, + { + VolumeName: "vol-a", + NodeID: "node-a", + CommittedLSN: 15, + WALHeadLSN: 16, + Eligible: true, + }, + }) + if err != nil { + t.Fatalf("select candidate: %v", err) + } + if selected.NodeID != "node-a" { + t.Fatalf("selected node=%q, want node-a", selected.NodeID) + } +} + +func TestSelectPromotionCandidate_UsesWalHeadAsTiebreaker(t *testing.T) { + master := New(Config{}) + selected, err := master.SelectPromotionCandidate([]PromotionQueryResponse{ + { + VolumeName: "vol-a", + NodeID: "node-a", + CommittedLSN: 15, + WALHeadLSN: 16, + Eligible: true, + }, + { + VolumeName: "vol-a", + NodeID: "node-b", + CommittedLSN: 15, + WALHeadLSN: 19, + Eligible: true, + }, + }) + if err != nil { + t.Fatalf("select candidate: %v", err) + } + if selected.NodeID != "node-b" { + t.Fatalf("selected node=%q, want node-b", selected.NodeID) + } +} + +func TestSelectPromotionCandidate_RejectsWhenNoEligibleCandidates(t *testing.T) { + master := New(Config{}) + _, err := master.SelectPromotionCandidate([]PromotionQueryResponse{ + {NodeID: "node-a", Eligible: false, Reason: "needs_rebuild"}, + {NodeID: "node-b", Eligible: false, Reason: "epoch_mismatch"}, + }) + if err == nil { + t.Fatal("expected error when no eligible candidates") + } +} + +func TestAuthorizePromotion_ReassignsPrimaryAndAdvancesEpoch(t *testing.T) { + master := New(Config{}) + if err := master.DeclarePrimary(VolumeSpec{ + Name: "vol-a", + Path: "/tmp/vol-a.blk", + PrimaryNodeID: "node-a", + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + + assign, err := master.AuthorizePromotion("vol-a", []PromotionQueryResponse{ + { + VolumeName: "vol-a", + NodeID: "node-b", + CommittedLSN: 18, + WALHeadLSN: 19, + Eligible: true, + }, + { + VolumeName: "vol-a", + NodeID: "node-c", + CommittedLSN: 17, + WALHeadLSN: 20, + Eligible: true, + }, + }) + if err != nil { + t.Fatalf("authorize promotion: %v", err) + } + if assign.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", assign.NodeID) + } + if assign.Epoch != 2 { + t.Fatalf("assignment epoch=%d, want 2", assign.Epoch) + } + + view, ok := master.Volume("vol-a") + if !ok { + t.Fatal("master view missing") + } + if view.PrimaryNodeID != "node-b" { + t.Fatalf("view primary=%q, want node-b", view.PrimaryNodeID) + } + if view.DesiredEpoch != 2 { + t.Fatalf("view desired epoch=%d, want 2", view.DesiredEpoch) + } +} + +func TestAuthorizePromotion_RejectsMismatchedVolume(t *testing.T) { + master := New(Config{}) + if err := master.DeclarePrimary(VolumeSpec{ + Name: "vol-a", + Path: "/tmp/vol-a.blk", + PrimaryNodeID: "node-a", + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + + _, err := master.AuthorizePromotion("vol-a", []PromotionQueryResponse{ + { + VolumeName: "vol-b", + NodeID: "node-b", + CommittedLSN: 18, + WALHeadLSN: 19, + Eligible: true, + }, + }) + if err == nil { + t.Fatal("expected mismatched volume error") + } +} diff --git a/sw-block/runtime/protocolv2/identity.go b/sw-block/runtime/protocolv2/identity.go new file mode 100644 index 000000000..55ee2bf2a --- /dev/null +++ b/sw-block/runtime/protocolv2/identity.go @@ -0,0 +1,68 @@ +package protocolv2 + +import ( + "time" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ReplicaMember identifies one replica in the low-frequency identity loop. +// It carries identity and transport only, never replication progress. +type ReplicaMember struct { + NodeID string + DataAddr string + CtrlAddr string +} + +// Assignment is the Loop 1 assignment surface from masterv2 to volumev2. +type Assignment struct { + Name string + Path string + NodeID string + Epoch uint64 + LeaseTTL time.Duration + CreateOptions blockvol.CreateOptions + Role string + ReplicaSet []ReplicaMember +} + +// VolumeHeartbeat is the periodic identity-loop heartbeat surface. +// It stays small and carries only applied identity, compressed outward mode, +// and a passive committed-boundary cache. +type VolumeHeartbeat struct { + Name string + Path string + Epoch uint64 + Role string + Mode string + ModeReason string + CommittedLSN uint64 + RoleApplied bool + ReplicaReady bool +} + +// NodeHeartbeat is the low-frequency periodic heartbeat from a volume node. +type NodeHeartbeat struct { + NodeID string + ReportedAt time.Time + Volumes []VolumeHeartbeat +} + +// PromotionQueryRequest asks one candidate for fresh failover evidence. +type PromotionQueryRequest struct { + VolumeName string + ExpectedEpoch uint64 +} + +// PromotionQueryResponse returns fresh candidate evidence at query time. +type PromotionQueryResponse struct { + VolumeName string + NodeID string + Epoch uint64 + Role string + CommittedLSN uint64 + WALHeadLSN uint64 + ReceiverReady bool + Eligible bool + Reason string +} diff --git a/sw-block/runtime/protocolv2/recovery.go b/sw-block/runtime/protocolv2/recovery.go new file mode 100644 index 000000000..d4ceb7d24 --- /dev/null +++ b/sw-block/runtime/protocolv2/recovery.go @@ -0,0 +1,33 @@ +package protocolv2 + +// ReplicaSummaryRequest asks one node for a bounded takeover/reconstruction +// summary for a specific volume. This is richer than promotion evidence, but +// still smaller than full internal engine/session state. +type ReplicaSummaryRequest struct { + VolumeName string + ExpectedEpoch uint64 +} + +// ReplicaSummaryResponse is the bounded summary a future primary can use to +// reconstruct recovery truth. It preserves distinct boundary semantics without +// exposing raw shipper/session internals. +type ReplicaSummaryResponse struct { + VolumeName string + NodeID string + Epoch uint64 + Role string + Mode string + ModeReason string + RoleApplied bool + ReceiverReady bool + CommittedLSN uint64 + DurableLSN uint64 + CheckpointLSN uint64 + TargetLSN uint64 + AchievedLSN uint64 + RecoveryPhase string + LastBarrierOK bool + LastBarrierReason string + Eligible bool + Reason string +} diff --git a/sw-block/runtime/purev2/doc.go b/sw-block/runtime/purev2/doc.go new file mode 100644 index 000000000..388305103 --- /dev/null +++ b/sw-block/runtime/purev2/doc.go @@ -0,0 +1,4 @@ +// Package purev2 provides a small pure-V2 runtime shell that keeps +// semantic authority in the V2 engine while reusing blockvol execution +// mechanics. The initial slice is intentionally RF1/single-node only. +package purev2 diff --git a/sw-block/runtime/purev2/runtime.go b/sw-block/runtime/purev2/runtime.go new file mode 100644 index 000000000..edf84db73 --- /dev/null +++ b/sw-block/runtime/purev2/runtime.go @@ -0,0 +1,349 @@ +package purev2 + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "time" + + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" + "github.com/seaweedfs/seaweedfs/weed/server/blockcmd" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/v2bridge" +) + +const defaultListenAddr = "127.0.0.1:3260" + +// Config defines the small pure V2 runtime shell configuration. +type Config struct { + ListenAddr string + AdvertisedHost string + DiskType string +} + +// VolumeDebugSnapshot is the bounded outward runtime view for one volume. +type VolumeDebugSnapshot struct { + Path string + Info blockvol.VolumeInfo + Status blockvol.BlockVolumeStatus + Projection engine.PublicationProjection + HasProjection bool + CoreState engine.VolumeState + HasCoreState bool + ExecutedCommands []string +} + +// Runtime is a small pure-V2 process shell for RF1 local execution. +// It intentionally excludes master heartbeat, failover, and product-surface loops. +type Runtime struct { + store *storage.BlockVolumeStore + bindings *v2bridge.CommandBindings + core *engine.CoreEngine + dispatcher *blockcmd.Dispatcher + config Config + + mu sync.RWMutex + projections map[string]engine.PublicationProjection + executed map[string][]string +} + +// New creates a new RF1-oriented pure V2 runtime shell. +func New(cfg Config) *Runtime { + if cfg.ListenAddr == "" { + cfg.ListenAddr = defaultListenAddr + } + rt := &Runtime{ + store: storage.NewBlockVolumeStore(), + core: engine.NewCoreEngine(), + config: cfg, + projections: make(map[string]engine.PublicationProjection), + executed: make(map[string][]string), + } + rt.bindings = v2bridge.NewCommandBindings(rt.store, cfg.ListenAddr, cfg.AdvertisedHost) + rt.dispatcher = blockcmd.NewDispatcher( + blockcmd.NewServiceOps(runtimeBackend{runtime: rt}, nil, rt, nil), + blockcmd.NewHostEffects(rt.recordCommand, rt.emitCoreEvent, rt, rt), + ) + return rt +} + +// CreateVolume creates a new local block volume and registers it in the runtime store. +func (rt *Runtime) CreateVolume(path string, opts blockvol.CreateOptions, cfgs ...blockvol.BlockVolConfig) error { + if rt == nil { + return fmt.Errorf("purev2: runtime is nil") + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("purev2: create parent dir for %s: %w", path, err) + } + vol, err := blockvol.CreateBlockVol(path, opts, cfgs...) + if err != nil { + return fmt.Errorf("purev2: create volume %s: %w", path, err) + } + if err := vol.Close(); err != nil { + return fmt.Errorf("purev2: close created volume %s: %w", path, err) + } + _, err = rt.store.AddBlockVolume(path, rt.config.DiskType, cfgs...) + if err != nil { + return fmt.Errorf("purev2: register created volume %s: %w", path, err) + } + return nil +} + +// OpenVolume opens an existing volume into the runtime store. +func (rt *Runtime) OpenVolume(path string, cfgs ...blockvol.BlockVolConfig) error { + if rt == nil { + return fmt.Errorf("purev2: runtime is nil") + } + if _, err := rt.store.AddBlockVolume(path, rt.config.DiskType, cfgs...); err != nil { + return fmt.Errorf("purev2: open volume %s: %w", path, err) + } + return nil +} + +// ApplyPrimaryAssignment injects a static RF1 primary assignment into the pure runtime. +func (rt *Runtime) ApplyPrimaryAssignment(path string, epoch uint64, leaseTTL time.Duration) error { + return rt.applyAssignment(blockvol.BlockVolumeAssignment{ + Path: path, + Epoch: epoch, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: blockvol.LeaseTTLToWire(leaseTTL), + }) +} + +// BootstrapPrimary creates a volume if needed and applies a primary RF1 assignment. +func (rt *Runtime) BootstrapPrimary(path string, opts blockvol.CreateOptions, epoch uint64, leaseTTL time.Duration, cfgs ...blockvol.BlockVolConfig) error { + if _, ok := rt.store.GetBlockVolume(path); !ok { + if _, err := os.Stat(path); err == nil { + if err := rt.OpenVolume(path, cfgs...); err != nil { + return err + } + } else { + if err := rt.CreateVolume(path, opts, cfgs...); err != nil { + return err + } + } + } + return rt.ApplyPrimaryAssignment(path, epoch, leaseTTL) +} + +// WriteLBA writes data at the given logical block address. +func (rt *Runtime) WriteLBA(path string, lba uint64, data []byte) error { + var status blockvol.V2StatusSnapshot + err := rt.store.WithVolume(path, func(vol *blockvol.BlockVol) error { + if err := vol.WriteLBA(lba, data); err != nil { + return err + } + status = vol.StatusSnapshot() + return nil + }) + if err != nil { + return err + } + rt.observeLocalBoundaries(path, status) + return nil +} + +// ReadLBA reads data at the given logical block address. +func (rt *Runtime) ReadLBA(path string, lba uint64, length uint32) ([]byte, error) { + var out []byte + err := rt.store.WithVolume(path, func(vol *blockvol.BlockVol) error { + data, err := vol.ReadLBA(lba, length) + if err != nil { + return err + } + out = data + return nil + }) + return out, err +} + +// SyncCache forces durable local flush through the reused blockvol backend. +func (rt *Runtime) SyncCache(path string) error { + var ( + status blockvol.V2StatusSnapshot + attempt bool + ) + err := rt.store.WithVolume(path, func(vol *blockvol.BlockVol) error { + attempt = true + if err := vol.SyncCache(); err != nil { + return err + } + status = vol.StatusSnapshot() + return nil + }) + if err != nil { + if attempt { + rt.emitCoreEvent(engine.BarrierRejected{ID: path, Reason: err.Error()}) + } + return err + } + rt.observeLocalBoundaries(path, status) + rt.emitCoreEvent(engine.BarrierAccepted{ID: path, FlushedLSN: status.CommittedLSN}) + rt.emitCoreEvent(engine.CheckpointAdvanced{ID: path, CheckpointLSN: status.CheckpointLSN}) + return nil +} + +// Snapshot returns the bounded runtime and core view for one volume. +func (rt *Runtime) Snapshot(path string) (VolumeDebugSnapshot, error) { + if rt == nil { + return VolumeDebugSnapshot{}, fmt.Errorf("purev2: runtime is nil") + } + var snap VolumeDebugSnapshot + err := rt.store.WithVolume(path, func(vol *blockvol.BlockVol) error { + snap.Path = path + snap.Info = vol.Info() + snap.Status = vol.Status() + return nil + }) + if err != nil { + return VolumeDebugSnapshot{}, err + } + + if proj, ok := rt.Projection(path); ok { + snap.HasProjection = true + snap.Projection = proj + } + if st, ok := rt.core.State(path); ok { + snap.HasCoreState = true + snap.CoreState = st + } + + rt.mu.RLock() + snap.ExecutedCommands = append([]string(nil), rt.executed[path]...) + rt.mu.RUnlock() + return snap, nil +} + +// WithVolume exposes controlled access to one underlying block volume. +// It exists so higher-level runtimes can attach frontend adapters without +// importing weed/server lifecycle code. +func (rt *Runtime) WithVolume(path string, fn func(*blockvol.BlockVol) error) error { + if rt == nil { + return fmt.Errorf("purev2: runtime is nil") + } + return rt.store.WithVolume(path, fn) +} + +// Projection implements blockcmd.ProjectionReader. +func (rt *Runtime) Projection(volumeID string) (engine.PublicationProjection, bool) { + rt.mu.RLock() + defer rt.mu.RUnlock() + proj, ok := rt.projections[volumeID] + return proj, ok +} + +// StoreProjection implements blockcmd.ProjectionCacheWriter. +func (rt *Runtime) StoreProjection(volumeID string, projection engine.PublicationProjection) { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.projections[volumeID] = projection +} + +// Close closes all registered volumes. +func (rt *Runtime) Close() { + if rt == nil { + return + } + rt.store.Close() +} + +func (rt *Runtime) applyAssignment(a blockvol.BlockVolumeAssignment) error { + ev, ok := assignmentEvent(a) + if !ok { + return nil + } + result := rt.core.ApplyEvent(ev) + return rt.dispatcher.Run(result.Commands, &a) +} + +func (rt *Runtime) emitCoreEvent(ev engine.Event) { + result := rt.core.ApplyEvent(ev) + _ = rt.dispatcher.Run(result.Commands, nil) +} + +func (rt *Runtime) recordCommand(volumeID, name string) { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.executed[volumeID] = append(rt.executed[volumeID], name) +} + +func (rt *Runtime) observeLocalBoundaries(path string, status blockvol.V2StatusSnapshot) { + rt.emitCoreEvent(engine.CommittedLSNAdvanced{ + ID: path, + CommittedLSN: status.CommittedLSN, + }) +} + +func assignmentEvent(a blockvol.BlockVolumeAssignment) (engine.AssignmentDelivered, bool) { + ev := engine.AssignmentDelivered{ + ID: a.Path, + Epoch: a.Epoch, + } + switch blockvol.RoleFromWire(a.Role) { + case blockvol.RolePrimary: + ev.Role = engine.RolePrimary + return ev, true + case blockvol.RoleReplica: + ev.Role = engine.RoleReplica + return ev, true + case blockvol.RoleRebuilding: + ev.Role = engine.RoleReplica + return ev, true + default: + return engine.AssignmentDelivered{}, false + } +} + +type runtimeBackend struct { + runtime *Runtime +} + +func (ops runtimeBackend) ApplyRole(assignment blockvol.BlockVolumeAssignment) (bool, error) { + if ops.runtime == nil || ops.runtime.bindings == nil { + return false, nil + } + if err := ops.runtime.bindings.ApplyRole(assignment); err != nil { + return false, err + } + ops.runtime.emitCoreEvent(engine.RoleApplied{ID: assignment.Path}) + return true, nil +} + +func (ops runtimeBackend) StartReceiver(assignment blockvol.BlockVolumeAssignment) (bool, error) { + if ops.runtime == nil || ops.runtime.bindings == nil { + return false, nil + } + if assignment.ReplicaDataAddr == "" || assignment.ReplicaCtrlAddr == "" { + return false, nil + } + if _, err := ops.runtime.bindings.StartReceiver(assignment.Path, assignment.ReplicaDataAddr, assignment.ReplicaCtrlAddr); err != nil { + return false, err + } + return true, nil +} + +func (ops runtimeBackend) ConfigureShipper(volumeID string, replicas []engine.ReplicaAssignment) (bool, bool, error) { + if ops.runtime == nil || ops.runtime.bindings == nil || len(replicas) == 0 { + return false, false, nil + } + addrs := make([]blockvol.ReplicaAddr, 0, len(replicas)) + for _, replica := range replicas { + if replica.Endpoint.DataAddr == "" || replica.Endpoint.CtrlAddr == "" { + continue + } + addrs = append(addrs, blockvol.ReplicaAddr{ + ServerID: replica.ReplicaID, + DataAddr: replica.Endpoint.DataAddr, + CtrlAddr: replica.Endpoint.CtrlAddr, + }) + } + if len(addrs) == 0 { + return false, false, nil + } + if _, err := ops.runtime.bindings.ConfigurePrimaryReplication(volumeID, addrs); err != nil { + return false, false, err + } + return true, ops.runtime.bindings.IsPrimaryShipperConnected(volumeID), nil +} diff --git a/sw-block/runtime/purev2/runtime_test.go b/sw-block/runtime/purev2/runtime_test.go new file mode 100644 index 000000000..c276065be --- /dev/null +++ b/sw-block/runtime/purev2/runtime_test.go @@ -0,0 +1,154 @@ +package purev2 + +import ( + "bytes" + "path/filepath" + "reflect" + "testing" + "time" + + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +func TestRuntime_RF1BootstrapCreateAssignSnapshot(t *testing.T) { + tempDir := t.TempDir() + rt := New(Config{}) + defer rt.Close() + + path := filepath.Join(tempDir, "rf1-primary.blk") + if err := rt.BootstrapPrimary(path, testCreateOptions(), 1, 30*time.Second); err != nil { + t.Fatalf("bootstrap primary: %v", err) + } + + snap, err := rt.Snapshot(path) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if snap.Status.Role != blockvol.RolePrimary { + t.Fatalf("role=%v", snap.Status.Role) + } + if snap.Status.Epoch != 1 { + t.Fatalf("epoch=%d", snap.Status.Epoch) + } + if !snap.HasProjection { + t.Fatal("expected published projection") + } + if snap.Projection.Role != engine.RolePrimary { + t.Fatalf("projection role=%v", snap.Projection.Role) + } + if snap.Projection.Mode.Name != engine.ModeAllocatedOnly { + t.Fatalf("projection mode=%s", snap.Projection.Mode.Name) + } + if snap.Projection.Publication.Reason != "allocated_only" { + t.Fatalf("publication reason=%q", snap.Projection.Publication.Reason) + } + if !snap.Projection.Readiness.RoleApplied { + t.Fatal("role_applied should be true after apply_role") + } + if !reflect.DeepEqual(snap.ExecutedCommands, []string{"apply_role"}) { + t.Fatalf("executed=%v", snap.ExecutedCommands) + } +} + +func TestRuntime_RF1RestartPreservesLocalData(t *testing.T) { + tempDir := t.TempDir() + path := filepath.Join(tempDir, "rf1-restart.blk") + payload := bytes.Repeat([]byte{0x5A}, 4096) + + func() { + rt := New(Config{}) + defer rt.Close() + + if err := rt.BootstrapPrimary(path, testCreateOptions(), 1, 30*time.Second); err != nil { + t.Fatalf("bootstrap primary: %v", err) + } + if err := rt.WriteLBA(path, 0, payload); err != nil { + t.Fatalf("write: %v", err) + } + if err := rt.SyncCache(path); err != nil { + t.Fatalf("sync cache: %v", err) + } + }() + + rt := New(Config{}) + defer rt.Close() + + if err := rt.OpenVolume(path); err != nil { + t.Fatalf("open volume: %v", err) + } + readBack, err := rt.ReadLBA(path, 0, uint32(len(payload))) + if err != nil { + t.Fatalf("read after restart: %v", err) + } + if !bytes.Equal(readBack, payload) { + t.Fatal("readback mismatch after restart") + } + + if err := rt.ApplyPrimaryAssignment(path, 1, 30*time.Second); err != nil { + t.Fatalf("re-apply primary: %v", err) + } + snap, err := rt.Snapshot(path) + if err != nil { + t.Fatalf("snapshot after reopen: %v", err) + } + if !snap.HasCoreState { + t.Fatal("expected core state after re-apply") + } + if snap.CoreState.Readiness.RoleApplied != true { + t.Fatal("role_applied should remain true after reopen assignment") + } +} + +func TestRuntime_LocalBoundaryObservationsAdvanceCoreState(t *testing.T) { + tempDir := t.TempDir() + rt := New(Config{}) + defer rt.Close() + + path := filepath.Join(tempDir, "rf1-boundaries.blk") + if err := rt.BootstrapPrimary(path, testCreateOptions(), 1, 30*time.Second); err != nil { + t.Fatalf("bootstrap primary: %v", err) + } + + payload := bytes.Repeat([]byte{0x33}, 4096) + if err := rt.WriteLBA(path, 0, payload); err != nil { + t.Fatalf("write: %v", err) + } + + snap, err := rt.Snapshot(path) + if err != nil { + t.Fatalf("snapshot after write: %v", err) + } + if !snap.HasCoreState { + t.Fatal("expected core state after write") + } + if snap.CoreState.Boundary.CommittedLSN == 0 { + t.Fatalf("committed_lsn=%d, want > 0", snap.CoreState.Boundary.CommittedLSN) + } + if snap.CoreState.Boundary.DurableLSN != 0 { + t.Fatalf("durable_lsn=%d before sync, want 0", snap.CoreState.Boundary.DurableLSN) + } + + if err := rt.SyncCache(path); err != nil { + t.Fatalf("sync cache: %v", err) + } + + snap, err = rt.Snapshot(path) + if err != nil { + t.Fatalf("snapshot after sync: %v", err) + } + if snap.CoreState.Boundary.DurableLSN < snap.CoreState.Boundary.CommittedLSN { + t.Fatalf("durable_lsn=%d committed_lsn=%d", snap.CoreState.Boundary.DurableLSN, snap.CoreState.Boundary.CommittedLSN) + } + if snap.CoreState.Boundary.CheckpointLSN > snap.CoreState.Boundary.DurableLSN { + t.Fatalf("checkpoint_lsn=%d durable_lsn=%d", snap.CoreState.Boundary.CheckpointLSN, snap.CoreState.Boundary.DurableLSN) + } +} + +func testCreateOptions() blockvol.CreateOptions { + return blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + } +} diff --git a/sw-block/runtime/volumev2/control_session.go b/sw-block/runtime/volumev2/control_session.go new file mode 100644 index 000000000..2e2e3de8b --- /dev/null +++ b/sw-block/runtime/volumev2/control_session.go @@ -0,0 +1,45 @@ +package volumev2 + +import ( + "fmt" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" +) + +// ControlSession is the minimal control-plane contract between volumev2 and masterv2. +type ControlSession interface { + Heartbeat(masterv2.NodeHeartbeat) ([]masterv2.Assignment, error) +} + +// PromotionEvidenceSource is the on-demand Loop 1 query surface used during +// failover arbitration. +type PromotionEvidenceSource interface { + QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) +} + +// InProcessSession is the first in-process control-plane adapter used by the MVP. +type InProcessSession struct { + master *masterv2.Master +} + +// NewInProcessSession creates a control session backed by one in-process masterv2. +func NewInProcessSession(master *masterv2.Master) (*InProcessSession, error) { + if master == nil { + return nil, fmt.Errorf("volumev2: master is nil") + } + return &InProcessSession{master: master}, nil +} + +// Heartbeat sends one periodic heartbeat to masterv2 and returns the assignments to apply. +func (s *InProcessSession) Heartbeat(hb masterv2.NodeHeartbeat) ([]masterv2.Assignment, error) { + if s == nil || s.master == nil { + return nil, fmt.Errorf("volumev2: control session is nil") + } + return s.master.HandleHeartbeat(hb) +} + +// Sync is retained as a narrow compatibility shim for existing tests while the +// three-channel Loop 1 surface settles. +func (s *InProcessSession) Sync(hb masterv2.NodeHeartbeat) ([]masterv2.Assignment, error) { + return s.Heartbeat(hb) +} diff --git a/sw-block/runtime/volumev2/dataplane.go b/sw-block/runtime/volumev2/dataplane.go new file mode 100644 index 000000000..807acb5ab --- /dev/null +++ b/sw-block/runtime/volumev2/dataplane.go @@ -0,0 +1,83 @@ +package volumev2 + +import ( + "fmt" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/purev2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// DataPlane is the minimal single-node execution contract for volumev2. +// It keeps backend mechanics replaceable while volumev2 owns the control shell. +type DataPlane interface { + BootstrapPrimary(path string, opts blockvol.CreateOptions, epoch uint64, leaseTTL time.Duration) error + WriteLBA(path string, lba uint64, data []byte) error + ReadLBA(path string, lba uint64, length uint32) ([]byte, error) + SyncCache(path string) error + Snapshot(path string) (purev2.VolumeDebugSnapshot, error) + WithVolume(path string, fn func(*blockvol.BlockVol) error) error + Close() +} + +// PureRuntimeDataPlane adapts the current purev2 runtime as a volumev2 data plane. +type PureRuntimeDataPlane struct { + runtime *purev2.Runtime +} + +// NewPureRuntimeDataPlane wraps one purev2 runtime behind the DataPlane contract. +func NewPureRuntimeDataPlane(runtime *purev2.Runtime) (*PureRuntimeDataPlane, error) { + if runtime == nil { + return nil, fmt.Errorf("volumev2: pure runtime is nil") + } + return &PureRuntimeDataPlane{runtime: runtime}, nil +} + +func (dp *PureRuntimeDataPlane) BootstrapPrimary(path string, opts blockvol.CreateOptions, epoch uint64, leaseTTL time.Duration) error { + if dp == nil || dp.runtime == nil { + return fmt.Errorf("volumev2: data plane is nil") + } + return dp.runtime.BootstrapPrimary(path, opts, epoch, leaseTTL) +} + +func (dp *PureRuntimeDataPlane) WriteLBA(path string, lba uint64, data []byte) error { + if dp == nil || dp.runtime == nil { + return fmt.Errorf("volumev2: data plane is nil") + } + return dp.runtime.WriteLBA(path, lba, data) +} + +func (dp *PureRuntimeDataPlane) ReadLBA(path string, lba uint64, length uint32) ([]byte, error) { + if dp == nil || dp.runtime == nil { + return nil, fmt.Errorf("volumev2: data plane is nil") + } + return dp.runtime.ReadLBA(path, lba, length) +} + +func (dp *PureRuntimeDataPlane) SyncCache(path string) error { + if dp == nil || dp.runtime == nil { + return fmt.Errorf("volumev2: data plane is nil") + } + return dp.runtime.SyncCache(path) +} + +func (dp *PureRuntimeDataPlane) Snapshot(path string) (purev2.VolumeDebugSnapshot, error) { + if dp == nil || dp.runtime == nil { + return purev2.VolumeDebugSnapshot{}, fmt.Errorf("volumev2: data plane is nil") + } + return dp.runtime.Snapshot(path) +} + +func (dp *PureRuntimeDataPlane) WithVolume(path string, fn func(*blockvol.BlockVol) error) error { + if dp == nil || dp.runtime == nil { + return fmt.Errorf("volumev2: data plane is nil") + } + return dp.runtime.WithVolume(path, fn) +} + +func (dp *PureRuntimeDataPlane) Close() { + if dp == nil || dp.runtime == nil { + return + } + dp.runtime.Close() +} diff --git a/sw-block/runtime/volumev2/failover.go b/sw-block/runtime/volumev2/failover.go new file mode 100644 index 000000000..4c9758065 --- /dev/null +++ b/sw-block/runtime/volumev2/failover.go @@ -0,0 +1,305 @@ +package volumev2 + +import ( + "fmt" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" +) + +// FailoverParticipant is the minimal surface needed to execute one failover +// flow across Loop 1 authorization and Loop 2 takeover preparation. +type FailoverParticipant interface { + QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) + QueryReplicaSummarySource + PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) + GatePrimaryActivation(volumeName string, truth ReconstructedPrimaryTruth) error +} + +// QueryReplicaSummarySource aliases the peer summary surface so the failover +// helper can reuse the existing bounded takeover contract. +type QueryReplicaSummarySource interface { + ReplicaSummarySource +} + +// FailoverResult captures the outputs of one authorized and prepared failover. +type FailoverResult struct { + Candidate masterv2.PromotionQueryResponse + Assignment masterv2.Assignment + Truth ReconstructedPrimaryTruth +} + +// FailoverStage is the coarse external progress marker for one failover +// session. It is intended for orchestration and debugging, not semantics. +type FailoverStage string + +const ( + FailoverStageNew FailoverStage = "new" + FailoverStageEvidenceCollected FailoverStage = "evidence_collected" + FailoverStageAuthorized FailoverStage = "authorized" + FailoverStagePrepared FailoverStage = "prepared" + FailoverStageActivated FailoverStage = "activated" + FailoverStageFailed FailoverStage = "failed" +) + +// FailoverSnapshot is a read-only summary of the session's current observable +// state for drivers, tests, and debug surfaces. +type FailoverSnapshot struct { + VolumeName string + ExpectedEpoch uint64 + Stage FailoverStage + LastError string + ResponseCount int + SelectedNodeID string + Result FailoverResult +} + +// FailoverSession is a thin orchestration object that exposes the narrow +// failover stages explicitly so higher-level drivers can stop after +// authorization, inspect intermediate results, or run the whole sequence. +type FailoverSession struct { + master *masterv2.Master + volumeName string + expectedEpoch uint64 + participants []FailoverParticipant + + responses []masterv2.PromotionQueryResponse + byNode map[string]FailoverParticipant + result FailoverResult + stage FailoverStage + lastErr error +} + +// NewFailoverSession validates the narrow failover inputs and returns a +// stepwise orchestration session. +func NewFailoverSession(master *masterv2.Master, volumeName string, expectedEpoch uint64, participants []FailoverParticipant) (*FailoverSession, error) { + if master == nil { + return nil, fmt.Errorf("volumev2: master is nil") + } + if volumeName == "" { + return nil, fmt.Errorf("volumev2: volume name is required") + } + if len(participants) == 0 { + return nil, fmt.Errorf("volumev2: failover participants are required") + } + return &FailoverSession{ + master: master, + volumeName: volumeName, + expectedEpoch: expectedEpoch, + participants: participants, + stage: FailoverStageNew, + }, nil +} + +// CollectPromotionEvidence gathers fresh promotion responses from all +// configured participants. +func (s *FailoverSession) CollectPromotionEvidence() ([]masterv2.PromotionQueryResponse, error) { + if s == nil { + return nil, fmt.Errorf("volumev2: failover session is nil") + } + responses := make([]masterv2.PromotionQueryResponse, 0, len(s.participants)) + byNode := make(map[string]FailoverParticipant, len(s.participants)) + for _, participant := range s.participants { + if participant == nil { + continue + } + resp, err := participant.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ + VolumeName: s.volumeName, + ExpectedEpoch: s.expectedEpoch, + }) + if err != nil { + return nil, s.failf("volumev2: promotion evidence %s: %w", s.volumeName, err) + } + if resp.NodeID == "" { + return nil, s.failf("volumev2: promotion evidence for %s missing node id", s.volumeName) + } + responses = append(responses, resp) + byNode[resp.NodeID] = participant + } + if len(responses) == 0 { + return nil, s.failf("volumev2: no failover evidence collected for %s", s.volumeName) + } + s.responses = responses + s.byNode = byNode + s.stage = FailoverStageEvidenceCollected + s.lastErr = nil + return append([]masterv2.PromotionQueryResponse(nil), responses...), nil +} + +// Authorize asks masterv2 to pick and authorize the new primary assignment from +// the collected fresh promotion evidence. +func (s *FailoverSession) Authorize() (masterv2.Assignment, error) { + if s == nil { + return masterv2.Assignment{}, fmt.Errorf("volumev2: failover session is nil") + } + if len(s.responses) == 0 { + if _, err := s.CollectPromotionEvidence(); err != nil { + return masterv2.Assignment{}, err + } + } + assignment, err := s.master.AuthorizePromotion(s.volumeName, s.responses) + if err != nil { + return masterv2.Assignment{}, s.fail(err) + } + s.result.Assignment = assignment + for _, resp := range s.responses { + if resp.NodeID == assignment.NodeID { + s.result.Candidate = resp + break + } + } + s.stage = FailoverStageAuthorized + s.lastErr = nil + return assignment, nil +} + +// PrepareTakeover runs the selected node's bounded takeover reconstruction. +func (s *FailoverSession) PrepareTakeover() (ReconstructedPrimaryTruth, error) { + if s == nil { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: failover session is nil") + } + if s.result.Assignment.NodeID == "" { + if _, err := s.Authorize(); err != nil { + return ReconstructedPrimaryTruth{}, err + } + } + selected, ok := s.byNode[s.result.Assignment.NodeID] + if !ok { + return ReconstructedPrimaryTruth{}, s.failf("volumev2: authorized node %q missing participant", s.result.Assignment.NodeID) + } + peers := make([]ReplicaSummarySource, 0, len(s.byNode)-1) + for nodeID, participant := range s.byNode { + if nodeID == s.result.Assignment.NodeID { + continue + } + peers = append(peers, participant) + } + truth, err := selected.PreparePrimaryTakeover(PrimaryTakeoverPlan{ + Assignment: s.result.Assignment, + Peers: peers, + }) + s.result.Truth = truth + if err != nil { + return truth, s.fail(err) + } + s.stage = FailoverStagePrepared + s.lastErr = nil + return truth, nil +} + +// Activate gates the selected primary on the reconstructed takeover truth. +func (s *FailoverSession) Activate() error { + if s == nil { + return fmt.Errorf("volumev2: failover session is nil") + } + if s.result.Assignment.NodeID == "" { + if _, err := s.Authorize(); err != nil { + return err + } + } + if s.result.Truth.PrimaryNodeID == "" { + if _, err := s.PrepareTakeover(); err != nil { + return err + } + } + selected, ok := s.byNode[s.result.Assignment.NodeID] + if !ok { + return s.failf("volumev2: authorized node %q missing participant", s.result.Assignment.NodeID) + } + if err := selected.GatePrimaryActivation(s.volumeName, s.result.Truth); err != nil { + return s.fail(err) + } + s.stage = FailoverStageActivated + s.lastErr = nil + return nil +} + +// Result returns the latest collected candidate, assignment, and reconstructed +// takeover truth known to the session. +func (s *FailoverSession) Result() FailoverResult { + if s == nil { + return FailoverResult{} + } + return s.result +} + +// Stage returns the current coarse failover stage. +func (s *FailoverSession) Stage() FailoverStage { + if s == nil { + return FailoverStageFailed + } + return s.stage +} + +// LastError returns the last stage error observed by the session. +func (s *FailoverSession) LastError() error { + if s == nil { + return fmt.Errorf("volumev2: failover session is nil") + } + return s.lastErr +} + +// Snapshot returns a stable read-only view of the session's externally useful +// state. +func (s *FailoverSession) Snapshot() FailoverSnapshot { + if s == nil { + return FailoverSnapshot{Stage: FailoverStageFailed, LastError: "volumev2: failover session is nil"} + } + lastErr := "" + if s.lastErr != nil { + lastErr = s.lastErr.Error() + } + return FailoverSnapshot{ + VolumeName: s.volumeName, + ExpectedEpoch: s.expectedEpoch, + Stage: s.stage, + LastError: lastErr, + ResponseCount: len(s.responses), + SelectedNodeID: s.result.Assignment.NodeID, + Result: s.result, + } +} + +// Run executes the full failover path from fresh evidence collection through +// activation gating. +func (s *FailoverSession) Run() (FailoverResult, error) { + if s == nil { + return FailoverResult{}, fmt.Errorf("volumev2: failover session is nil") + } + if _, err := s.CollectPromotionEvidence(); err != nil { + return s.Result(), err + } + if _, err := s.Authorize(); err != nil { + return s.Result(), err + } + if _, err := s.PrepareTakeover(); err != nil { + return s.Result(), err + } + if err := s.Activate(); err != nil { + return s.Result(), err + } + return s.Result(), nil +} + +// ExecuteFailoverFlow runs the narrow failover path: +// fresh promotion evidence -> master authorization -> takeover preparation -> +// activation gate. It intentionally does not choreograph catch-up or rebuild. +func ExecuteFailoverFlow(master *masterv2.Master, volumeName string, expectedEpoch uint64, participants []FailoverParticipant) (FailoverResult, error) { + session, err := NewFailoverSession(master, volumeName, expectedEpoch, participants) + if err != nil { + return FailoverResult{}, err + } + return session.Run() +} + +func (s *FailoverSession) fail(err error) error { + if s == nil { + return err + } + s.stage = FailoverStageFailed + s.lastErr = err + return err +} + +func (s *FailoverSession) failf(format string, args ...any) error { + return s.fail(fmt.Errorf(format, args...)) +} diff --git a/sw-block/runtime/volumev2/failover_driver.go b/sw-block/runtime/volumev2/failover_driver.go new file mode 100644 index 000000000..4e00ca426 --- /dev/null +++ b/sw-block/runtime/volumev2/failover_driver.go @@ -0,0 +1,122 @@ +package volumev2 + +import ( + "fmt" + "slices" + "sync" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" +) + +// InProcessFailoverDriver is the first thin driver that wires one in-process +// masterv2 instance to a set of failover-capable participants. It owns no +// recovery logic; it only resolves participants and constructs sessions. +type InProcessFailoverDriver struct { + master *masterv2.Master + + mu sync.RWMutex + participants map[string]FailoverParticipant +} + +// NewInProcessFailoverDriver creates a driver for one in-process masterv2. +func NewInProcessFailoverDriver(master *masterv2.Master) (*InProcessFailoverDriver, error) { + if master == nil { + return nil, fmt.Errorf("volumev2: master is nil") + } + return &InProcessFailoverDriver{ + master: master, + participants: make(map[string]FailoverParticipant), + }, nil +} + +// RegisterParticipant binds one stable node id to one failover participant. +func (d *InProcessFailoverDriver) RegisterParticipant(nodeID string, participant FailoverParticipant) error { + if d == nil { + return fmt.Errorf("volumev2: failover driver is nil") + } + if nodeID == "" { + return fmt.Errorf("volumev2: participant node id is required") + } + if participant == nil { + return fmt.Errorf("volumev2: participant %q is nil", nodeID) + } + d.mu.Lock() + defer d.mu.Unlock() + d.participants[nodeID] = participant + return nil +} + +// UnregisterParticipant removes one node from the in-process driver. +func (d *InProcessFailoverDriver) UnregisterParticipant(nodeID string) { + if d == nil || nodeID == "" { + return + } + d.mu.Lock() + defer d.mu.Unlock() + delete(d.participants, nodeID) +} + +// ParticipantNodeIDs returns the currently registered node ids in stable order. +func (d *InProcessFailoverDriver) ParticipantNodeIDs() []string { + if d == nil { + return nil + } + d.mu.RLock() + defer d.mu.RUnlock() + nodeIDs := make([]string, 0, len(d.participants)) + for nodeID := range d.participants { + nodeIDs = append(nodeIDs, nodeID) + } + slices.Sort(nodeIDs) + return nodeIDs +} + +// NewSession constructs a FailoverSession using either the requested node ids +// or all currently registered participants when none are specified. +func (d *InProcessFailoverDriver) NewSession(volumeName string, expectedEpoch uint64, nodeIDs ...string) (*FailoverSession, error) { + if d == nil { + return nil, fmt.Errorf("volumev2: failover driver is nil") + } + participants, err := d.resolveParticipants(nodeIDs) + if err != nil { + return nil, err + } + return NewFailoverSession(d.master, volumeName, expectedEpoch, participants) +} + +// Execute runs one failover using the resolved participant set. +func (d *InProcessFailoverDriver) Execute(volumeName string, expectedEpoch uint64, nodeIDs ...string) (FailoverResult, error) { + session, err := d.NewSession(volumeName, expectedEpoch, nodeIDs...) + if err != nil { + return FailoverResult{}, err + } + return session.Run() +} + +func (d *InProcessFailoverDriver) resolveParticipants(nodeIDs []string) ([]FailoverParticipant, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + if len(d.participants) == 0 { + return nil, fmt.Errorf("volumev2: no failover participants registered") + } + + resolvedIDs := nodeIDs + if len(resolvedIDs) == 0 { + resolvedIDs = make([]string, 0, len(d.participants)) + for nodeID := range d.participants { + resolvedIDs = append(resolvedIDs, nodeID) + } + slices.Sort(resolvedIDs) + } + + participants := make([]FailoverParticipant, 0, len(resolvedIDs)) + for _, nodeID := range resolvedIDs { + participant, ok := d.participants[nodeID] + if !ok { + return nil, fmt.Errorf("volumev2: unknown failover participant %q", nodeID) + } + participants = append(participants, participant) + } + return participants, nil +} diff --git a/sw-block/runtime/volumev2/frontend.go b/sw-block/runtime/volumev2/frontend.go new file mode 100644 index 000000000..068864b8d --- /dev/null +++ b/sw-block/runtime/volumev2/frontend.go @@ -0,0 +1,92 @@ +package volumev2 + +import ( + "fmt" + "io" + "log" + "net" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" +) + +// ISCSITargetExport is a small handle for one running iSCSI frontend export. +type ISCSITargetExport struct { + iqn string + addr string + server *iscsi.TargetServer +} + +// IQN returns the exported target name. +func (e *ISCSITargetExport) IQN() string { + if e == nil { + return "" + } + return e.iqn +} + +// Address returns the current listen address. +func (e *ISCSITargetExport) Address() string { + if e == nil { + return "" + } + return e.addr +} + +// Close stops the iSCSI target server. +func (e *ISCSITargetExport) Close() error { + if e == nil || e.server == nil { + return nil + } + return e.server.Close() +} + +// ExportISCSI starts a small iSCSI target server for one named volume. +// This gives the single-node MVP a real block frontend without depending on weed/server. +func (n *Node) ExportISCSI(name, listenAddr, iqn string) (*ISCSITargetExport, error) { + if n == nil { + return nil, fmt.Errorf("volumev2: node is nil") + } + if listenAddr == "" { + listenAddr = "127.0.0.1:0" + } + if iqn == "" { + iqn = "iqn.2026-04.com.seaweedfs:v2." + name + } + path, err := n.pathFor(name) + if err != nil { + return nil, err + } + + var dev iscsi.BlockDevice + if err := n.dataPlane.WithVolume(path, func(vol *blockvol.BlockVol) error { + dev = blockvol.NewBlockVolAdapter(vol) + return nil + }); err != nil { + return nil, err + } + if dev == nil { + return nil, fmt.Errorf("volumev2: no block device for %q", name) + } + + cfg := iscsi.DefaultTargetConfig() + cfg.TargetName = iqn + logger := log.New(io.Discard, "", 0) + server := iscsi.NewTargetServer(listenAddr, cfg, logger) + server.AddVolume(iqn, dev) + + ln, err := net.Listen("tcp", listenAddr) + if err != nil { + return nil, fmt.Errorf("volumev2: iscsi listen %s: %w", listenAddr, err) + } + server.SetPortalAddr(ln.Addr().String() + ",1") + go func() { + _ = server.Serve(ln) + }() + + return &ISCSITargetExport{ + iqn: iqn, + addr: ln.Addr().String(), + server: server, + }, nil +} diff --git a/sw-block/runtime/volumev2/frontend_test.go b/sw-block/runtime/volumev2/frontend_test.go new file mode 100644 index 000000000..80b1994f8 --- /dev/null +++ b/sw-block/runtime/volumev2/frontend_test.go @@ -0,0 +1,233 @@ +package volumev2 + +import ( + "bytes" + "encoding/binary" + "net" + "path/filepath" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" +) + +func TestVolumeV2_ExportISCSI_StartsFrontendAndAcceptsLogin(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + path := filepath.Join(t.TempDir(), "frontend-vol.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "frontend-vol", + Path: path, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 2: %v", err) + } + + export, err := node.ExportISCSI("frontend-vol", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.frontend-vol") + if err != nil { + t.Fatalf("export iscsi: %v", err) + } + defer export.Close() + + conn, err := net.DialTimeout("tcp", export.Address(), 2*time.Second) + if err != nil { + t.Fatalf("dial target: %v", err) + } + defer conn.Close() + + params := iscsi.NewParams() + params.Set("InitiatorName", "iqn.2026-04.com.seaweedfs:initiator.test") + params.Set("TargetName", export.IQN()) + params.Set("SessionType", "Normal") + + loginReq := &iscsi.PDU{} + loginReq.SetOpcode(iscsi.OpLoginReq) + loginReq.SetLoginStages(iscsi.StageSecurityNeg, iscsi.StageFullFeature) + loginReq.SetLoginTransit(true) + loginReq.SetISID([6]byte{0x00, 0x02, 0x3D, 0x00, 0x00, 0x01}) + loginReq.SetCmdSN(1) + loginReq.DataSegment = params.Encode() + + if err := iscsi.WritePDU(conn, loginReq); err != nil { + t.Fatalf("write login req: %v", err) + } + resp, err := iscsi.ReadPDU(conn) + if err != nil { + t.Fatalf("read login resp: %v", err) + } + if resp.LoginStatusClass() != iscsi.LoginStatusSuccess { + t.Fatalf("login failed: %d/%d", resp.LoginStatusClass(), resp.LoginStatusDetail()) + } +} + +func TestKernelDataPlaneClosure_ISCSIWriteReadVerify(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + path := filepath.Join(t.TempDir(), "data-plane-vol.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "data-plane-vol", + Path: path, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 2: %v", err) + } + + export, err := node.ExportISCSI("data-plane-vol", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.data-plane-vol") + if err != nil { + t.Fatalf("export iscsi: %v", err) + } + defer export.Close() + + conn := mustLoginISCSI(t, export.Address(), export.IQN()) + defer conn.Close() + + writeData := make([]byte, 4096) + for i := range writeData { + writeData[i] = byte((i * 7) % 251) + } + + var writeCDB [16]byte + writeCDB[0] = iscsi.ScsiWrite10 + binary.BigEndian.PutUint32(writeCDB[2:6], 0) + binary.BigEndian.PutUint16(writeCDB[7:9], 1) + resp := sendSCSICmd(t, conn, writeCDB, 2, false, true, writeData, uint32(len(writeData))) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("iscsi write failed: status=%d", resp.SCSIStatus()) + } + + var syncCDB [16]byte + syncCDB[0] = iscsi.ScsiSyncCache10 + resp = sendSCSICmd(t, conn, syncCDB, 3, false, false, nil, 0) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("iscsi sync cache failed: status=%d", resp.SCSIStatus()) + } + + var readCDB [16]byte + readCDB[0] = iscsi.ScsiRead10 + binary.BigEndian.PutUint32(readCDB[2:6], 0) + binary.BigEndian.PutUint16(readCDB[7:9], 1) + resp = sendSCSICmd(t, conn, readCDB, 4, true, false, nil, uint32(len(writeData))) + if resp.Opcode() != iscsi.OpSCSIDataIn { + t.Fatalf("expected Data-In, got %s", iscsi.OpcodeName(resp.Opcode())) + } + if !bytes.Equal(resp.DataSegment, writeData) { + t.Fatal("iscsi readback mismatch") + } + + readBack, err := node.ReadLBA("data-plane-vol", 0, uint32(len(writeData))) + if err != nil { + t.Fatalf("backend read: %v", err) + } + if !bytes.Equal(readBack, writeData) { + t.Fatal("backend readback mismatch") + } +} + +func mustLoginISCSI(t *testing.T, addr, iqn string) net.Conn { + t.Helper() + + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + t.Fatalf("dial target: %v", err) + } + + params := iscsi.NewParams() + params.Set("InitiatorName", "iqn.2026-04.com.seaweedfs:initiator.test") + params.Set("TargetName", iqn) + params.Set("SessionType", "Normal") + + loginReq := &iscsi.PDU{} + loginReq.SetOpcode(iscsi.OpLoginReq) + loginReq.SetLoginStages(iscsi.StageSecurityNeg, iscsi.StageFullFeature) + loginReq.SetLoginTransit(true) + loginReq.SetISID([6]byte{0x00, 0x02, 0x3D, 0x00, 0x00, 0x01}) + loginReq.SetCmdSN(1) + loginReq.DataSegment = params.Encode() + + if err := iscsi.WritePDU(conn, loginReq); err != nil { + conn.Close() + t.Fatalf("write login req: %v", err) + } + resp, err := iscsi.ReadPDU(conn) + if err != nil { + conn.Close() + t.Fatalf("read login resp: %v", err) + } + if resp.LoginStatusClass() != iscsi.LoginStatusSuccess { + conn.Close() + t.Fatalf("login failed: %d/%d", resp.LoginStatusClass(), resp.LoginStatusDetail()) + } + return conn +} + +func sendSCSICmd(t *testing.T, conn net.Conn, cdb [16]byte, cmdSN uint32, read bool, write bool, dataOut []byte, expLen uint32) *iscsi.PDU { + t.Helper() + + cmd := &iscsi.PDU{} + cmd.SetOpcode(iscsi.OpSCSICmd) + flags := uint8(iscsi.FlagF) + if read { + flags |= iscsi.FlagR + } + if write { + flags |= iscsi.FlagW + } + cmd.SetOpSpecific1(flags) + cmd.SetInitiatorTaskTag(cmdSN) + cmd.SetExpectedDataTransferLength(expLen) + cmd.SetCmdSN(cmdSN) + cmd.SetCDB(cdb) + if dataOut != nil { + cmd.DataSegment = dataOut + } + + if err := iscsi.WritePDU(conn, cmd); err != nil { + t.Fatalf("write scsi cmd: %v", err) + } + resp, err := iscsi.ReadPDU(conn) + if err != nil { + t.Fatalf("read scsi resp: %v", err) + } + return resp +} diff --git a/sw-block/runtime/volumev2/orchestrator.go b/sw-block/runtime/volumev2/orchestrator.go new file mode 100644 index 000000000..426ef10e4 --- /dev/null +++ b/sw-block/runtime/volumev2/orchestrator.go @@ -0,0 +1,37 @@ +package volumev2 + +import "fmt" + +// Orchestrator closes the MVP control loop: +// heartbeat -> assignments -> local apply. +type Orchestrator struct { + node *Node + session ControlSession +} + +// NewOrchestrator creates a small volumev2 control/data orchestrator. +func NewOrchestrator(node *Node, session ControlSession) (*Orchestrator, error) { + if node == nil { + return nil, fmt.Errorf("volumev2: node is nil") + } + if session == nil { + return nil, fmt.Errorf("volumev2: control session is nil") + } + return &Orchestrator{node: node, session: session}, nil +} + +// SyncOnce reports one heartbeat and applies any returned assignments. +func (o *Orchestrator) SyncOnce() error { + if o == nil || o.node == nil || o.session == nil { + return fmt.Errorf("volumev2: orchestrator is not initialized") + } + hb, err := o.node.Heartbeat() + if err != nil { + return err + } + assignments, err := o.session.Heartbeat(hb) + if err != nil { + return err + } + return o.node.ApplyAssignments(assignments) +} diff --git a/sw-block/runtime/volumev2/poc_test.go b/sw-block/runtime/volumev2/poc_test.go new file mode 100644 index 000000000..58fbfb1cc --- /dev/null +++ b/sw-block/runtime/volumev2/poc_test.go @@ -0,0 +1,1242 @@ +package volumev2 + +import ( + "bytes" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +func TestPOC_MasterV2VolumeV2_RF1HeartbeatAssignmentFlow(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volumev2 node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + path := filepath.Join(t.TempDir(), "rf1-poc.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "vol-a", + Path: path, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + + assignments, err := session.Sync(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("preview sync: %v", err) + } + if len(assignments) != 1 { + t.Fatalf("preview assignments=%d, want 1", len(assignments)) + } + if assignments[0].Epoch != 1 { + t.Fatalf("epoch=%d, want 1", assignments[0].Epoch) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync once 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync once 2: %v", err) + } + + view, ok := master.Volume("vol-a") + if !ok { + t.Fatal("master view missing") + } + if view.ObservedRole != "primary" { + t.Fatalf("observed role=%q", view.ObservedRole) + } + if !view.RoleApplied { + t.Fatal("role_applied should be true") + } + if view.Mode != "allocated_only" { + t.Fatalf("mode=%q", view.Mode) + } + + payload := bytes.Repeat([]byte{0x4A}, 4096) + if err := node.WriteLBA("vol-a", 0, payload); err != nil { + t.Fatalf("write: %v", err) + } + if err := node.SyncCache("vol-a"); err != nil { + t.Fatalf("sync cache: %v", err) + } + readBack, err := node.ReadLBA("vol-a", 0, uint32(len(payload))) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(readBack, payload) { + t.Fatal("payload mismatch") + } +} + +func TestPOC_MasterV2VolumeV2_ReissuesAssignmentAfterDesiredPathChange(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volumev2 node: %v", err) + } + defer node.Close() + + tempDir := t.TempDir() + path1 := filepath.Join(tempDir, "vol-a-1.blk") + path2 := filepath.Join(tempDir, "vol-a-2.blk") + + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "vol-a", + Path: path1, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary path1: %v", err) + } + + assignments, err := master.HandleHeartbeat(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("handle heartbeat 0: %v", err) + } + if err := node.ApplyAssignments(assignments); err != nil { + t.Fatalf("apply assignments path1: %v", err) + } + + if _, err := master.HandleHeartbeat(mustHeartbeat(t, node)); err != nil { + t.Fatalf("handle heartbeat after path1: %v", err) + } + + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "vol-a", + Path: path2, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary path2: %v", err) + } + + assignments, err = master.HandleHeartbeat(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("handle heartbeat for path change: %v", err) + } + if len(assignments) != 1 { + t.Fatalf("assignments=%d, want 1", len(assignments)) + } + if assignments[0].Epoch != 2 { + t.Fatalf("epoch=%d, want 2", assignments[0].Epoch) + } + if assignments[0].Path != path2 { + t.Fatalf("path=%q, want %q", assignments[0].Path, path2) + } + + if err := node.ApplyAssignments(assignments); err != nil { + t.Fatalf("apply assignments path2: %v", err) + } + assignments, err = master.HandleHeartbeat(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("handle heartbeat after path2: %v", err) + } + if len(assignments) != 0 { + t.Fatalf("unexpected reissue count=%d", len(assignments)) + } + + view, ok := master.Volume("vol-a") + if !ok { + t.Fatal("master view missing") + } + if view.Path != path2 { + t.Fatalf("view path=%q, want %q", view.Path, path2) + } + if view.ObservedEpoch != 2 { + t.Fatalf("observed epoch=%d, want 2", view.ObservedEpoch) + } +} + +func TestPOC_MasterV2VolumeV2_OrchestratorSyncsSingleNodeMVP(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volumev2 node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + path := filepath.Join(t.TempDir(), "vol-orch.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "vol-orch", + Path: path, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync once 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync once 2: %v", err) + } + + snap, err := node.Snapshot("vol-orch") + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if !snap.HasProjection || !snap.Projection.Readiness.RoleApplied { + t.Fatal("expected applied projection after orchestrator sync") + } +} + +func TestKernelControlPlaneClosure_ConvergesAndReissuesOnDesiredChange(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volumev2 node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + tempDir := t.TempDir() + path1 := filepath.Join(tempDir, "control-a.blk") + path2 := filepath.Join(tempDir, "control-b.blk") + spec := masterv2.VolumeSpec{ + Name: "control-vol", + Path: path1, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + } + if err := master.DeclarePrimary(spec); err != nil { + t.Fatalf("declare primary path1: %v", err) + } + + assignments, err := session.Sync(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("preview sync before convergence: %v", err) + } + if len(assignments) != 1 { + t.Fatalf("preview assignments=%d, want 1", len(assignments)) + } + if assignments[0].Path != path1 || assignments[0].Epoch != 1 { + t.Fatalf("unexpected first assignment: %+v", assignments[0]) + } + + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 2: %v", err) + } + + assignments, err = session.Sync(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("post-convergence sync: %v", err) + } + if len(assignments) != 0 { + t.Fatalf("unexpected post-convergence assignments=%d", len(assignments)) + } + + view, ok := master.Volume("control-vol") + if !ok { + t.Fatal("master view missing after convergence") + } + if view.DesiredEpoch != 1 || view.ObservedEpoch != 1 { + t.Fatalf("epochs after convergence desired=%d observed=%d", view.DesiredEpoch, view.ObservedEpoch) + } + if view.ObservedRole != "primary" || !view.RoleApplied { + t.Fatalf("view after convergence = %+v", view) + } + + spec.Path = path2 + if err := master.DeclarePrimary(spec); err != nil { + t.Fatalf("declare primary path2: %v", err) + } + assignments, err = session.Sync(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("sync after desired change: %v", err) + } + if len(assignments) != 1 { + t.Fatalf("assignments after desired change=%d, want 1", len(assignments)) + } + if assignments[0].Path != path2 || assignments[0].Epoch != 2 { + t.Fatalf("unexpected reissued assignment: %+v", assignments[0]) + } + + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 3: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 4: %v", err) + } + + assignments, err = session.Sync(mustHeartbeat(t, node)) + if err != nil { + t.Fatalf("final sync after re-convergence: %v", err) + } + if len(assignments) != 0 { + t.Fatalf("unexpected assignments after re-convergence=%d", len(assignments)) + } + + view, ok = master.Volume("control-vol") + if !ok { + t.Fatal("master view missing after desired change") + } + if view.Path != path2 || view.DesiredEpoch != 2 || view.ObservedEpoch != 2 { + t.Fatalf("view after desired change = %+v", view) + } + if view.ObservedRole != "primary" || !view.RoleApplied { + t.Fatalf("final view not applied = %+v", view) + } +} + +func TestLoop1_PromotionQueryUsesFreshEvidenceSeparateFromHeartbeat(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volumev2 node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + path := filepath.Join(t.TempDir(), "query-vol.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "query-vol", + Path: path, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 2: %v", err) + } + + hb := mustHeartbeat(t, node) + if len(hb.Volumes) != 1 { + t.Fatalf("heartbeat volumes=%d, want 1", len(hb.Volumes)) + } + if hb.Volumes[0].Mode != "allocated_only" { + t.Fatalf("heartbeat mode=%q", hb.Volumes[0].Mode) + } + + before, err := node.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ + VolumeName: "query-vol", + ExpectedEpoch: 1, + }) + if err != nil { + t.Fatalf("query before write: %v", err) + } + if !before.Eligible { + t.Fatalf("before write should be eligible: %+v", before) + } + if hb.Volumes[0].CommittedLSN != before.CommittedLSN { + t.Fatalf("heartbeat committed_lsn=%d, query committed_lsn=%d", hb.Volumes[0].CommittedLSN, before.CommittedLSN) + } + + payload := bytes.Repeat([]byte{0x7D}, 4096) + if err := node.WriteLBA("query-vol", 0, payload); err != nil { + t.Fatalf("write: %v", err) + } + if err := node.SyncCache("query-vol"); err != nil { + t.Fatalf("sync cache: %v", err) + } + + after, err := node.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ + VolumeName: "query-vol", + ExpectedEpoch: 1, + }) + if err != nil { + t.Fatalf("query after write: %v", err) + } + if after.CommittedLSN <= before.CommittedLSN { + t.Fatalf("committed_lsn before=%d after=%d", before.CommittedLSN, after.CommittedLSN) + } + if after.WALHeadLSN <= before.WALHeadLSN { + t.Fatalf("wal_head_lsn before=%d after=%d", before.WALHeadLSN, after.WALHeadLSN) + } + if after.CommittedLSN <= hb.Volumes[0].CommittedLSN { + t.Fatalf("fresh query committed_lsn=%d should advance beyond heartbeat cache=%d", after.CommittedLSN, hb.Volumes[0].CommittedLSN) + } + + if _, err := master.HandleHeartbeat(mustHeartbeat(t, node)); err != nil { + t.Fatalf("refresh heartbeat after write: %v", err) + } + view, ok := master.Volume("query-vol") + if !ok { + t.Fatal("master view missing after refreshed heartbeat") + } + if view.CommittedLSN != after.CommittedLSN { + t.Fatalf("master cached committed_lsn=%d, want %d", view.CommittedLSN, after.CommittedLSN) + } +} + +func TestLoop2_ReplicaSummaryPreservesBoundaryLayers(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + node, err := New(Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volumev2 node: %v", err) + } + defer node.Close() + session, err := NewInProcessSession(master) + if err != nil { + t.Fatalf("new session: %v", err) + } + orchestrator, err := NewOrchestrator(node, session) + if err != nil { + t.Fatalf("new orchestrator: %v", err) + } + + path := filepath.Join(t.TempDir(), "summary-vol.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "summary-vol", + Path: path, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 1: %v", err) + } + if err := orchestrator.SyncOnce(); err != nil { + t.Fatalf("sync 2: %v", err) + } + + payload := bytes.Repeat([]byte{0x22}, 4096) + if err := node.WriteLBA("summary-vol", 0, payload); err != nil { + t.Fatalf("write: %v", err) + } + + beforeSync, err := node.QueryReplicaSummary(protocolv2.ReplicaSummaryRequest{ + VolumeName: "summary-vol", + ExpectedEpoch: 1, + }) + if err != nil { + t.Fatalf("summary before sync: %v", err) + } + if !beforeSync.Eligible { + t.Fatalf("summary before sync should be eligible: %+v", beforeSync) + } + if beforeSync.CommittedLSN == 0 { + t.Fatalf("committed_lsn=%d, want > 0", beforeSync.CommittedLSN) + } + if beforeSync.DurableLSN != 0 { + t.Fatalf("durable_lsn=%d before sync, want 0", beforeSync.DurableLSN) + } + + if err := node.SyncCache("summary-vol"); err != nil { + t.Fatalf("sync cache: %v", err) + } + + afterSync, err := node.QueryReplicaSummary(protocolv2.ReplicaSummaryRequest{ + VolumeName: "summary-vol", + ExpectedEpoch: 1, + }) + if err != nil { + t.Fatalf("summary after sync: %v", err) + } + if afterSync.CommittedLSN < beforeSync.CommittedLSN { + t.Fatalf("committed_lsn before=%d after=%d", beforeSync.CommittedLSN, afterSync.CommittedLSN) + } + if afterSync.DurableLSN < afterSync.CommittedLSN { + t.Fatalf("durable_lsn=%d committed_lsn=%d", afterSync.DurableLSN, afterSync.CommittedLSN) + } + if afterSync.CheckpointLSN > afterSync.DurableLSN { + t.Fatalf("checkpoint_lsn=%d durable_lsn=%d", afterSync.CheckpointLSN, afterSync.DurableLSN) + } + if afterSync.RecoveryPhase != "idle" { + t.Fatalf("recovery_phase=%q", afterSync.RecoveryPhase) + } +} + +func TestLoop2_ReplacementPrimaryReconstructsTakeoverTruth(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + + tempDir := t.TempDir() + assignments := []masterv2.Assignment{ + { + Name: "takeover-vol", + Path: filepath.Join(tempDir, "takeover-b.blk"), + NodeID: "node-b", + Epoch: 3, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }, + { + Name: "takeover-vol", + Path: filepath.Join(tempDir, "takeover-c.blk"), + NodeID: "node-c", + Epoch: 3, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }, + } + if err := nodeB.ApplyAssignments(assignments); err != nil { + t.Fatalf("apply node-b assignment: %v", err) + } + if err := nodeC.ApplyAssignments(assignments); err != nil { + t.Fatalf("apply node-c assignment: %v", err) + } + + payload := bytes.Repeat([]byte{0x6E}, 4096) + if err := nodeB.WriteLBA("takeover-vol", 0, payload); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("takeover-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + candidate, err := master.SelectPromotionCandidate([]masterv2.PromotionQueryResponse{ + mustPromotionEvidence(t, nodeB, "takeover-vol", 3), + mustPromotionEvidence(t, nodeC, "takeover-vol", 3), + }) + if err != nil { + t.Fatalf("select promotion candidate: %v", err) + } + if candidate.NodeID != "node-b" { + t.Fatalf("candidate node=%q, want node-b", candidate.NodeID) + } + + truth, err := nodeB.ReconstructTakeoverTruth("takeover-vol", 3, []ReplicaSummarySource{nodeC}) + if err != nil { + t.Fatalf("reconstruct takeover truth: %v", err) + } + if truth.PrimaryNodeID != "node-b" { + t.Fatalf("primary_node=%q, want node-b", truth.PrimaryNodeID) + } + if truth.ReplicaCount != 2 { + t.Fatalf("replica_count=%d, want 2", truth.ReplicaCount) + } + if truth.CommittedLSN == 0 { + t.Fatalf("committed_lsn=%d, want > 0", truth.CommittedLSN) + } + if truth.DurableLSN < truth.CommittedLSN { + t.Fatalf("durable_lsn=%d committed_lsn=%d", truth.DurableLSN, truth.CommittedLSN) + } + if truth.Degraded { + t.Fatalf("unexpected degraded truth: %+v", truth) + } +} + +func TestLoop2_PrepareAndGatePrimaryTakeover_AllowsHealthyCandidate(t *testing.T) { + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + + tempDir := t.TempDir() + assignB := masterv2.Assignment{ + Name: "takeover-allow-vol", + Path: filepath.Join(tempDir, "takeover-allow-b.blk"), + NodeID: "node-b", + Epoch: 5, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + } + assignC := masterv2.Assignment{ + Name: "takeover-allow-vol", + Path: filepath.Join(tempDir, "takeover-allow-c.blk"), + NodeID: "node-c", + Epoch: 5, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{assignC}); err != nil { + t.Fatalf("apply peer assignment: %v", err) + } + + payload := bytes.Repeat([]byte{0x2C}, 4096) + if err := nodeB.ApplyAssignments([]masterv2.Assignment{assignB}); err != nil { + t.Fatalf("seed candidate assignment: %v", err) + } + if err := nodeB.WriteLBA("takeover-allow-vol", 0, payload); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("takeover-allow-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + truth, err := nodeB.PreparePrimaryTakeover(PrimaryTakeoverPlan{ + Assignment: assignB, + Peers: []ReplicaSummarySource{nodeC}, + }) + if err != nil { + t.Fatalf("prepare primary takeover: %v", err) + } + if err := nodeB.GatePrimaryActivation(assignB.Name, truth); err != nil { + t.Fatalf("gate primary activation: %v", err) + } + if truth.PrimaryNodeID != "node-b" { + t.Fatalf("primary_node=%q, want node-b", truth.PrimaryNodeID) + } + if truth.ReplicaCount != 2 { + t.Fatalf("replica_count=%d, want 2", truth.ReplicaCount) + } +} + +func TestLoop2_ApplyPrimaryTakeover_GatesDegradedTruth(t *testing.T) { + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + + tempDir := t.TempDir() + assignB := masterv2.Assignment{ + Name: "takeover-gated-vol", + Path: filepath.Join(tempDir, "takeover-gated-b.blk"), + NodeID: "node-b", + Epoch: 6, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + } + + truth, err := nodeB.PreparePrimaryTakeover(PrimaryTakeoverPlan{ + Assignment: assignB, + Peers: []ReplicaSummarySource{ + staticReplicaSummarySource{resp: protocolv2.ReplicaSummaryResponse{ + VolumeName: "takeover-gated-vol", + NodeID: "node-c", + Epoch: 5, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }}, + }, + }) + if err != nil { + t.Fatalf("prepare primary takeover: %v", err) + } + if err := nodeB.GatePrimaryActivation(assignB.Name, truth); err == nil { + t.Fatal("expected takeover gate error") + } +} + +func TestFailoverFlow_AuthorizesAndActivatesHealthyCandidate(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "flow-b.blk") + pathC := filepath.Join(tempDir, "flow-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "flow-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "flow-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "flow-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + + payload := bytes.Repeat([]byte{0x5A}, 4096) + if err := nodeB.WriteLBA("flow-vol", 0, payload); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("flow-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + result, err := ExecuteFailoverFlow(master, "flow-vol", 2, []FailoverParticipant{nodeB, nodeC}) + if err != nil { + t.Fatalf("execute failover flow: %v", err) + } + if result.Candidate.NodeID != "node-b" { + t.Fatalf("candidate node=%q, want node-b", result.Candidate.NodeID) + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID) + } + if result.Assignment.Epoch != 2 { + t.Fatalf("assignment epoch=%d, want 2", result.Assignment.Epoch) + } + if result.Truth.PrimaryNodeID != "node-b" { + t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID) + } + if result.Truth.Degraded { + t.Fatalf("unexpected degraded truth: %+v", result.Truth) + } + + view, ok := master.Volume("flow-vol") + if !ok { + t.Fatal("master view missing") + } + if view.PrimaryNodeID != "node-b" { + t.Fatalf("view primary=%q, want node-b", view.PrimaryNodeID) + } + if view.DesiredEpoch != 2 { + t.Fatalf("view desired epoch=%d, want 2", view.DesiredEpoch) + } +} + +func TestFailoverFlow_StopsAtActivationGate(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "flow-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "flow-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "flow-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + result, err := ExecuteFailoverFlow(master, "flow-gated-vol", 2, []FailoverParticipant{ + nodeB, + staticFailoverParticipant{ + promotion: masterv2.PromotionQueryResponse{ + VolumeName: "flow-gated-vol", + NodeID: "node-c", + Epoch: 1, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + summary: protocolv2.ReplicaSummaryResponse{ + VolumeName: "flow-gated-vol", + NodeID: "node-c", + Epoch: 1, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + }, + }) + if err == nil { + t.Fatal("expected failover gate error") + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID) + } + if !result.Truth.Degraded { + t.Fatalf("expected degraded truth: %+v", result.Truth) + } + if !result.Truth.NeedsRebuild { + t.Fatalf("expected needs_rebuild truth: %+v", result.Truth) + } +} + +func TestFailoverSession_StepwiseStagesExposeIntermediateState(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "session-b.blk") + pathC := filepath.Join(tempDir, "session-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "session-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "session-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "session-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if err := nodeB.WriteLBA("session-vol", 0, bytes.Repeat([]byte{0x41}, 4096)); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("session-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + session, err := NewFailoverSession(master, "session-vol", 2, []FailoverParticipant{nodeB, nodeC}) + if err != nil { + t.Fatalf("new failover session: %v", err) + } + if session.Stage() != FailoverStageNew { + t.Fatalf("initial stage=%q, want %q", session.Stage(), FailoverStageNew) + } + + responses, err := session.CollectPromotionEvidence() + if err != nil { + t.Fatalf("collect promotion evidence: %v", err) + } + if len(responses) != 2 { + t.Fatalf("responses=%d, want 2", len(responses)) + } + if session.Stage() != FailoverStageEvidenceCollected { + t.Fatalf("stage after collect=%q, want %q", session.Stage(), FailoverStageEvidenceCollected) + } + if session.Snapshot().ResponseCount != 2 { + t.Fatalf("snapshot response_count=%d, want 2", session.Snapshot().ResponseCount) + } + + assign, err := session.Authorize() + if err != nil { + t.Fatalf("authorize: %v", err) + } + if assign.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", assign.NodeID) + } + if session.Stage() != FailoverStageAuthorized { + t.Fatalf("stage after authorize=%q, want %q", session.Stage(), FailoverStageAuthorized) + } + + truth, err := session.PrepareTakeover() + if err != nil { + t.Fatalf("prepare takeover: %v", err) + } + if truth.PrimaryNodeID != "node-b" { + t.Fatalf("truth primary=%q, want node-b", truth.PrimaryNodeID) + } + if session.Stage() != FailoverStagePrepared { + t.Fatalf("stage after prepare=%q, want %q", session.Stage(), FailoverStagePrepared) + } + + if err := session.Activate(); err != nil { + t.Fatalf("activate: %v", err) + } + if session.Stage() != FailoverStageActivated { + t.Fatalf("stage after activate=%q, want %q", session.Stage(), FailoverStageActivated) + } + + result := session.Result() + if result.Candidate.NodeID != "node-b" { + t.Fatalf("result candidate=%q, want node-b", result.Candidate.NodeID) + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("result assignment=%q, want node-b", result.Assignment.NodeID) + } + if result.Truth.PrimaryNodeID != "node-b" { + t.Fatalf("result truth primary=%q, want node-b", result.Truth.PrimaryNodeID) + } +} + +func TestFailoverSession_SnapshotCapturesFailureState(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "snapshot-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "snapshot-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "snapshot-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + session, err := NewFailoverSession(master, "snapshot-gated-vol", 2, []FailoverParticipant{ + nodeB, + staticFailoverParticipant{ + promotion: masterv2.PromotionQueryResponse{ + VolumeName: "snapshot-gated-vol", + NodeID: "node-c", + Epoch: 1, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + summary: protocolv2.ReplicaSummaryResponse{ + VolumeName: "snapshot-gated-vol", + NodeID: "node-c", + Epoch: 1, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + }, + }) + if err != nil { + t.Fatalf("new failover session: %v", err) + } + + if _, err := session.Run(); err == nil { + t.Fatal("expected failover session error") + } + if session.Stage() != FailoverStageFailed { + t.Fatalf("stage after failure=%q, want %q", session.Stage(), FailoverStageFailed) + } + if session.LastError() == nil { + t.Fatal("expected last error") + } + snap := session.Snapshot() + if snap.LastError == "" { + t.Fatal("expected snapshot last error") + } + if snap.SelectedNodeID != "node-b" { + t.Fatalf("snapshot selected node=%q, want node-b", snap.SelectedNodeID) + } + if !snap.Result.Truth.NeedsRebuild { + t.Fatalf("expected needs_rebuild truth in snapshot: %+v", snap.Result.Truth) + } +} + +func TestInProcessFailoverDriver_ExecuteHealthyFailover(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + driver, err := NewInProcessFailoverDriver(master) + if err != nil { + t.Fatalf("new failover driver: %v", err) + } + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := driver.RegisterParticipant("node-b", nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := driver.RegisterParticipant("node-c", nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "driver-b.blk") + pathC := filepath.Join(tempDir, "driver-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "driver-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "driver-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "driver-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if err := nodeB.WriteLBA("driver-vol", 0, bytes.Repeat([]byte{0x33}, 4096)); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("driver-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + result, err := driver.Execute("driver-vol", 2) + if err != nil { + t.Fatalf("driver execute: %v", err) + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID) + } + if result.Truth.PrimaryNodeID != "node-b" { + t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID) + } +} + +func TestInProcessFailoverDriver_ExecuteStopsOnGate(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + driver, err := NewInProcessFailoverDriver(master) + if err != nil { + t.Fatalf("new failover driver: %v", err) + } + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := driver.RegisterParticipant("node-b", nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := driver.RegisterParticipant("node-c", staticFailoverParticipant{ + promotion: masterv2.PromotionQueryResponse{ + VolumeName: "driver-gated-vol", + NodeID: "node-c", + Epoch: 1, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + summary: protocolv2.ReplicaSummaryResponse{ + VolumeName: "driver-gated-vol", + NodeID: "node-c", + Epoch: 1, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + }); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "driver-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "driver-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "driver-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + _, err = driver.Execute("driver-gated-vol", 2) + if err == nil { + t.Fatal("expected gated driver execute error") + } +} + +func mustHeartbeat(t *testing.T, node *Node) masterv2.NodeHeartbeat { + t.Helper() + hb, err := node.Heartbeat() + if err != nil { + t.Fatalf("heartbeat: %v", err) + } + return hb +} + +func mustPromotionEvidence(t *testing.T, node *Node, volumeName string, epoch uint64) masterv2.PromotionQueryResponse { + t.Helper() + resp, err := node.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ + VolumeName: volumeName, + ExpectedEpoch: epoch, + }) + if err != nil { + t.Fatalf("promotion evidence: %v", err) + } + return resp +} + +type staticReplicaSummarySource struct { + resp protocolv2.ReplicaSummaryResponse + err error +} + +func (s staticReplicaSummarySource) QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + return s.resp, s.err +} + +type staticFailoverParticipant struct { + promotion masterv2.PromotionQueryResponse + summary protocolv2.ReplicaSummaryResponse + err error +} + +func (s staticFailoverParticipant) QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { + return s.promotion, s.err +} + +func (s staticFailoverParticipant) QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + return s.summary, s.err +} + +func (s staticFailoverParticipant) PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) { + return ReconstructedPrimaryTruth{}, fmt.Errorf("static failover participant cannot prepare takeover") +} + +func (s staticFailoverParticipant) GatePrimaryActivation(string, ReconstructedPrimaryTruth) error { + return fmt.Errorf("static failover participant cannot gate activation") +} + +func testCreateOptions() blockvol.CreateOptions { + return blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + } +} diff --git a/sw-block/runtime/volumev2/reconstruction.go b/sw-block/runtime/volumev2/reconstruction.go new file mode 100644 index 000000000..74d3574d1 --- /dev/null +++ b/sw-block/runtime/volumev2/reconstruction.go @@ -0,0 +1,150 @@ +package volumev2 + +import ( + "fmt" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" +) + +// ReconstructedPrimaryTruth is the bounded truth a newly selected primary can +// derive from replica summaries before resuming data-control ownership. +type ReconstructedPrimaryTruth struct { + VolumeName string + PrimaryNodeID string + Epoch uint64 + CommittedLSN uint64 + DurableLSN uint64 + CheckpointLSN uint64 + TargetLSN uint64 + AchievedLSN uint64 + RecoveryPhase string + ReplicaCount int + Degraded bool + NeedsRebuild bool + Reason string +} + +// ReconstructPrimaryTruth derives a bounded recovery view for a newly chosen +// primary from the latest replica summaries. It intentionally stays smaller +// than the full internal engine/session graph and fail-closes on ambiguous +// epoch or recovery signals. +func ReconstructPrimaryTruth(primaryNodeID string, summaries []protocolv2.ReplicaSummaryResponse) (ReconstructedPrimaryTruth, error) { + if primaryNodeID == "" { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: primary node id is required") + } + if len(summaries) == 0 { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: replica summaries are required") + } + + var ( + selected protocolv2.ReplicaSummaryResponse + foundSelected bool + recoveryObserved bool + aggregateTarget uint64 + aggregateAchieved uint64 + ) + for _, summary := range summaries { + if summary.NodeID == primaryNodeID { + selected = summary + foundSelected = true + break + } + } + if !foundSelected { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: selected primary %q missing from summaries", primaryNodeID) + } + if !selected.Eligible { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: selected primary %q is not eligible: %s", primaryNodeID, selected.Reason) + } + + result := ReconstructedPrimaryTruth{ + VolumeName: selected.VolumeName, + PrimaryNodeID: selected.NodeID, + Epoch: selected.Epoch, + CommittedLSN: selected.CommittedLSN, + DurableLSN: selected.DurableLSN, + CheckpointLSN: selected.CheckpointLSN, + TargetLSN: selected.TargetLSN, + AchievedLSN: selected.AchievedLSN, + RecoveryPhase: selected.RecoveryPhase, + } + + for _, summary := range summaries { + if summary.VolumeName != selected.VolumeName { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: mixed volume summaries %q and %q", selected.VolumeName, summary.VolumeName) + } + if summary.Mode == "needs_rebuild" || summary.Reason == "needs_rebuild" { + result.NeedsRebuild = true + } + if !summary.LastBarrierOK && summary.LastBarrierReason != "" { + result.Degraded = true + if result.Reason == "" { + result.Reason = summary.LastBarrierReason + } + } + if summary.Epoch != selected.Epoch { + result.Degraded = true + result.Reason = "peer_epoch_mismatch" + continue + } + result.ReplicaCount++ + + if summary.CommittedLSN > result.CommittedLSN { + result.Degraded = true + result.Reason = "selected_not_most_recent" + } + if isRecoveryPhase(summary.RecoveryPhase) { + if !recoveryObserved { + aggregateTarget = summary.TargetLSN + aggregateAchieved = summary.AchievedLSN + recoveryObserved = true + } else { + if summary.TargetLSN > aggregateTarget { + aggregateTarget = summary.TargetLSN + } + if summary.AchievedLSN < aggregateAchieved { + aggregateAchieved = summary.AchievedLSN + } + } + result.RecoveryPhase = mergeRecoveryPhase(result.RecoveryPhase, summary.RecoveryPhase) + } + } + + if recoveryObserved { + result.TargetLSN = aggregateTarget + result.AchievedLSN = aggregateAchieved + } + if result.NeedsRebuild { + result.RecoveryPhase = "needs_rebuild" + } + return result, nil +} + +func isRecoveryPhase(phase string) bool { + switch phase { + case "catching_up", "rebuilding", "needs_rebuild": + return true + default: + return false + } +} + +func mergeRecoveryPhase(current, next string) string { + if recoveryRank(next) > recoveryRank(current) { + return next + } + return current +} + +func recoveryRank(phase string) int { + switch phase { + case "needs_rebuild": + return 3 + case "rebuilding": + return 2 + case "catching_up": + return 1 + default: + return 0 + } +} diff --git a/sw-block/runtime/volumev2/reconstruction_test.go b/sw-block/runtime/volumev2/reconstruction_test.go new file mode 100644 index 000000000..a96a37905 --- /dev/null +++ b/sw-block/runtime/volumev2/reconstruction_test.go @@ -0,0 +1,138 @@ +package volumev2 + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" +) + +func TestPrimaryLoss_ReconstructsBoundedTruthFromReplicaSummaries(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + candidate, err := master.SelectPromotionCandidate([]masterv2.PromotionQueryResponse{ + { + VolumeName: "vol-a", + NodeID: "node-b", + CommittedLSN: 15, + WALHeadLSN: 18, + Eligible: true, + }, + { + VolumeName: "vol-a", + NodeID: "node-c", + CommittedLSN: 12, + WALHeadLSN: 13, + Eligible: true, + }, + }) + if err != nil { + t.Fatalf("select candidate: %v", err) + } + + truth, err := ReconstructPrimaryTruth(candidate.NodeID, []protocolv2.ReplicaSummaryResponse{ + { + VolumeName: "vol-a", + NodeID: "node-b", + Epoch: 4, + Role: "replica", + Mode: "replica_ready", + CommittedLSN: 15, + DurableLSN: 15, + CheckpointLSN: 10, + TargetLSN: 40, + AchievedLSN: 30, + RecoveryPhase: "catching_up", + LastBarrierOK: true, + Eligible: true, + }, + { + VolumeName: "vol-a", + NodeID: "node-c", + Epoch: 4, + Role: "replica", + Mode: "replica_ready", + CommittedLSN: 12, + DurableLSN: 12, + CheckpointLSN: 8, + TargetLSN: 40, + AchievedLSN: 20, + RecoveryPhase: "catching_up", + LastBarrierOK: true, + Eligible: true, + }, + }) + if err != nil { + t.Fatalf("reconstruct truth: %v", err) + } + + if truth.PrimaryNodeID != "node-b" { + t.Fatalf("primary_node=%q, want node-b", truth.PrimaryNodeID) + } + if truth.CommittedLSN != 15 { + t.Fatalf("committed_lsn=%d, want 15", truth.CommittedLSN) + } + if truth.DurableLSN != 15 { + t.Fatalf("durable_lsn=%d, want 15", truth.DurableLSN) + } + if truth.CheckpointLSN != 10 { + t.Fatalf("checkpoint_lsn=%d, want 10", truth.CheckpointLSN) + } + if truth.TargetLSN != 40 { + t.Fatalf("target_lsn=%d, want 40", truth.TargetLSN) + } + if truth.AchievedLSN != 20 { + t.Fatalf("achieved_lsn=%d, want 20", truth.AchievedLSN) + } + if truth.RecoveryPhase != "catching_up" { + t.Fatalf("recovery_phase=%q, want catching_up", truth.RecoveryPhase) + } + if truth.Degraded { + t.Fatalf("unexpected degraded truth: %+v", truth) + } +} + +func TestPrimaryLoss_ReconstructionFailsClosedOnMismatchAndNeedsRebuild(t *testing.T) { + truth, err := ReconstructPrimaryTruth("node-b", []protocolv2.ReplicaSummaryResponse{ + { + VolumeName: "vol-a", + NodeID: "node-b", + Epoch: 5, + Role: "replica", + Mode: "replica_ready", + CommittedLSN: 15, + DurableLSN: 15, + CheckpointLSN: 10, + RecoveryPhase: "idle", + LastBarrierOK: true, + Eligible: true, + LastBarrierReason: "", + }, + { + VolumeName: "vol-a", + NodeID: "node-c", + Epoch: 4, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 12, + DurableLSN: 10, + CheckpointLSN: 8, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + }) + if err != nil { + t.Fatalf("reconstruct truth: %v", err) + } + if !truth.Degraded { + t.Fatalf("expected degraded truth: %+v", truth) + } + if !truth.NeedsRebuild { + t.Fatalf("expected needs_rebuild truth: %+v", truth) + } + if truth.RecoveryPhase != "needs_rebuild" { + t.Fatalf("recovery_phase=%q, want needs_rebuild", truth.RecoveryPhase) + } +} diff --git a/sw-block/runtime/volumev2/takeover.go b/sw-block/runtime/volumev2/takeover.go new file mode 100644 index 000000000..3c63fdc2d --- /dev/null +++ b/sw-block/runtime/volumev2/takeover.go @@ -0,0 +1,115 @@ +package volumev2 + +import ( + "fmt" + "slices" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" +) + +// ReplicaSummarySource is the bounded Loop 2 query surface a replacement +// primary uses to reconstruct takeover truth from peers. +type ReplicaSummarySource interface { + QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) +} + +// PrimaryTakeoverPlan is the minimal input needed for a selected replacement +// primary to prepare takeover locally. +type PrimaryTakeoverPlan struct { + Assignment masterv2.Assignment + Peers []ReplicaSummarySource +} + +// ReconstructTakeoverTruth lets the selected replacement primary gather its own +// bounded summary plus peer summaries before it resumes data-control ownership. +// Peers should exclude the current node; duplicate node IDs are ignored. +func (n *Node) ReconstructTakeoverTruth(volumeName string, expectedEpoch uint64, peers []ReplicaSummarySource) (ReconstructedPrimaryTruth, error) { + if n == nil { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: node is nil") + } + req := protocolv2.ReplicaSummaryRequest{ + VolumeName: volumeName, + ExpectedEpoch: expectedEpoch, + } + self, err := n.QueryReplicaSummary(req) + if err != nil { + return ReconstructedPrimaryTruth{}, err + } + + byNode := map[string]protocolv2.ReplicaSummaryResponse{ + self.NodeID: self, + } + for _, peer := range peers { + if peer == nil { + continue + } + summary, err := peer.QueryReplicaSummary(req) + if err != nil { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: peer replica summary %s: %w", volumeName, err) + } + if summary.NodeID == "" || summary.NodeID == n.id { + continue + } + byNode[summary.NodeID] = summary + } + + nodeIDs := make([]string, 0, len(byNode)) + for nodeID := range byNode { + nodeIDs = append(nodeIDs, nodeID) + } + slices.Sort(nodeIDs) + summaries := make([]protocolv2.ReplicaSummaryResponse, 0, len(nodeIDs)) + for _, nodeID := range nodeIDs { + summaries = append(summaries, byNode[nodeID]) + } + return ReconstructPrimaryTruth(n.id, summaries) +} + +// PreparePrimaryTakeover applies the local primary assignment and reconstructs +// bounded takeover truth from self and peers. It does not decide whether the +// new primary is allowed to activate data control yet. +func (n *Node) PreparePrimaryTakeover(plan PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) { + if n == nil { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: node is nil") + } + a := plan.Assignment + if a.Role != "primary" { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: unsupported takeover role %q", a.Role) + } + if a.NodeID != "" && a.NodeID != n.id { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: takeover assignment targets %q, node is %q", a.NodeID, n.id) + } + if err := n.ApplyAssignments([]masterv2.Assignment{a}); err != nil { + return ReconstructedPrimaryTruth{}, err + } + return n.ReconstructTakeoverTruth(a.Name, a.Epoch, plan.Peers) +} + +// GatePrimaryActivation fail-closes activation when the reconstructed truth +// says takeover is degraded, ambiguous, or rebuild-only. +func (n *Node) GatePrimaryActivation(volumeName string, truth ReconstructedPrimaryTruth) error { + if n == nil { + return fmt.Errorf("volumev2: node is nil") + } + if truth.NeedsRebuild { + return fmt.Errorf("volumev2: takeover gated for %s: needs rebuild", volumeName) + } + if truth.Degraded { + return fmt.Errorf("volumev2: takeover gated for %s: %s", volumeName, truth.Reason) + } + return nil +} + +// ApplyPrimaryTakeover is a narrow compatibility wrapper that prepares +// takeover truth and then gates activation. +func (n *Node) ApplyPrimaryTakeover(plan PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) { + truth, err := n.PreparePrimaryTakeover(plan) + if err != nil { + return ReconstructedPrimaryTruth{}, err + } + if err := n.GatePrimaryActivation(plan.Assignment.Name, truth); err != nil { + return truth, err + } + return truth, nil +} diff --git a/sw-block/runtime/volumev2/volume.go b/sw-block/runtime/volumev2/volume.go new file mode 100644 index 000000000..cdb5f92cb --- /dev/null +++ b/sw-block/runtime/volumev2/volume.go @@ -0,0 +1,312 @@ +package volumev2 + +import ( + "fmt" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/purev2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// Config defines the minimal volumev2 runtime identity. +type Config struct { + NodeID string + DataPlane DataPlane + Runtime *purev2.Runtime +} + +type volumeBinding struct { + name string + path string +} + +// Node is the minimal volumev2 runtime POC. +// It owns volume identity, local execution, and heartbeat reporting. +type Node struct { + id string + dataPlane DataPlane + + mu sync.RWMutex + volumes map[string]volumeBinding +} + +// New creates a minimal volumev2 node runtime. +func New(cfg Config) (*Node, error) { + if cfg.NodeID == "" { + return nil, fmt.Errorf("volumev2: node id is required") + } + dp := cfg.DataPlane + if dp == nil { + rt := cfg.Runtime + if rt == nil { + rt = purev2.New(purev2.Config{}) + } + var err error + dp, err = NewPureRuntimeDataPlane(rt) + if err != nil { + return nil, err + } + } + return &Node{ + id: cfg.NodeID, + dataPlane: dp, + volumes: make(map[string]volumeBinding), + }, nil +} + +// NodeID returns the stable runtime identity. +func (n *Node) NodeID() string { + if n == nil { + return "" + } + return n.id +} + +// ApplyAssignments applies the control messages emitted by masterv2. +func (n *Node) ApplyAssignments(assignments []masterv2.Assignment) error { + if n == nil { + return fmt.Errorf("volumev2: node is nil") + } + for _, assignment := range assignments { + if assignment.NodeID != "" && assignment.NodeID != n.id { + continue + } + if assignment.Role != "primary" { + return fmt.Errorf("volumev2: unsupported role %q", assignment.Role) + } + if err := n.dataPlane.BootstrapPrimary( + assignment.Path, + assignment.CreateOptions, + assignment.Epoch, + assignment.LeaseTTL, + ); err != nil { + return fmt.Errorf("volumev2: apply assignment %s: %w", assignment.Name, err) + } + n.mu.Lock() + n.volumes[assignment.Name] = volumeBinding{name: assignment.Name, path: assignment.Path} + n.mu.Unlock() + } + return nil +} + +// Heartbeat reports the minimal local runtime state back to masterv2. +func (n *Node) Heartbeat() (masterv2.NodeHeartbeat, error) { + if n == nil { + return masterv2.NodeHeartbeat{}, fmt.Errorf("volumev2: node is nil") + } + n.mu.RLock() + bindings := make([]volumeBinding, 0, len(n.volumes)) + for _, binding := range n.volumes { + bindings = append(bindings, binding) + } + n.mu.RUnlock() + + report := masterv2.NodeHeartbeat{ + NodeID: n.id, + ReportedAt: time.Now(), + Volumes: make([]masterv2.VolumeHeartbeat, 0, len(bindings)), + } + for _, binding := range bindings { + snap, err := n.dataPlane.Snapshot(binding.path) + if err != nil { + return masterv2.NodeHeartbeat{}, fmt.Errorf("volumev2: snapshot %s: %w", binding.name, err) + } + var committedLSN uint64 + if err := n.dataPlane.WithVolume(binding.path, func(vol *blockvol.BlockVol) error { + committedLSN = vol.StatusSnapshot().CommittedLSN + return nil + }); err != nil { + return masterv2.NodeHeartbeat{}, fmt.Errorf("volumev2: committed snapshot %s: %w", binding.name, err) + } + vol := masterv2.VolumeHeartbeat{ + Name: binding.name, + Path: binding.path, + Epoch: snap.Status.Epoch, + Role: snap.Status.Role.String(), + CommittedLSN: committedLSN, + } + if snap.HasProjection { + vol.Mode = string(snap.Projection.Mode.Name) + vol.ModeReason = snap.Projection.Mode.Reason + vol.RoleApplied = snap.Projection.Readiness.RoleApplied + vol.ReplicaReady = snap.Projection.Readiness.ReplicaReady + } + report.Volumes = append(report.Volumes, vol) + } + return report, nil +} + +// QueryPromotionEvidence returns fresh failover evidence outside the periodic +// heartbeat path. This keeps promotion arbitration separate from liveness. +func (n *Node) QueryPromotionEvidence(req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { + path, err := n.pathFor(req.VolumeName) + if err != nil { + return masterv2.PromotionQueryResponse{}, err + } + snap, err := n.dataPlane.Snapshot(path) + if err != nil { + return masterv2.PromotionQueryResponse{}, fmt.Errorf("volumev2: snapshot %s: %w", req.VolumeName, err) + } + resp := masterv2.PromotionQueryResponse{ + VolumeName: req.VolumeName, + NodeID: n.id, + Epoch: snap.Status.Epoch, + Role: snap.Status.Role.String(), + } + if err := n.dataPlane.WithVolume(path, func(vol *blockvol.BlockVol) error { + status := vol.StatusSnapshot() + resp.CommittedLSN = status.CommittedLSN + resp.WALHeadLSN = status.WALHeadLSN + return nil + }); err != nil { + return masterv2.PromotionQueryResponse{}, fmt.Errorf("volumev2: promotion evidence %s: %w", req.VolumeName, err) + } + if snap.HasProjection { + resp.ReceiverReady = snap.Projection.Readiness.ReplicaReady + switch { + case req.ExpectedEpoch != 0 && resp.Epoch != req.ExpectedEpoch: + resp.Reason = "epoch_mismatch" + case !snap.Projection.Readiness.RoleApplied: + resp.Reason = "role_not_applied" + case snap.Projection.Mode.Name == "needs_rebuild": + resp.Reason = "needs_rebuild" + default: + resp.Eligible = true + } + } else if req.ExpectedEpoch != 0 && resp.Epoch != req.ExpectedEpoch { + resp.Reason = "epoch_mismatch" + } else { + resp.Eligible = true + } + return resp, nil +} + +// QueryReplicaSummary returns a bounded takeover/reconstruction summary for one +// volume. It preserves distinct LSN semantics while avoiding raw internal +// shipper/session detail. +func (n *Node) QueryReplicaSummary(req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + path, err := n.pathFor(req.VolumeName) + if err != nil { + return protocolv2.ReplicaSummaryResponse{}, err + } + snap, err := n.dataPlane.Snapshot(path) + if err != nil { + return protocolv2.ReplicaSummaryResponse{}, fmt.Errorf("volumev2: snapshot %s: %w", req.VolumeName, err) + } + resp := protocolv2.ReplicaSummaryResponse{ + VolumeName: req.VolumeName, + NodeID: n.id, + Epoch: snap.Status.Epoch, + Role: snap.Status.Role.String(), + } + var status blockvol.V2StatusSnapshot + if err := n.dataPlane.WithVolume(path, func(vol *blockvol.BlockVol) error { + status = vol.StatusSnapshot() + return nil + }); err != nil { + return protocolv2.ReplicaSummaryResponse{}, fmt.Errorf("volumev2: replica summary %s: %w", req.VolumeName, err) + } + resp.CommittedLSN = status.CommittedLSN + resp.CheckpointLSN = status.CheckpointLSN + + if snap.HasProjection { + resp.Mode = string(snap.Projection.Mode.Name) + resp.ModeReason = snap.Projection.Mode.Reason + resp.RoleApplied = snap.Projection.Readiness.RoleApplied + resp.ReceiverReady = snap.Projection.Readiness.ReplicaReady + resp.DurableLSN = snap.Projection.Boundary.DurableLSN + resp.TargetLSN = snap.Projection.Boundary.TargetLSN + resp.AchievedLSN = snap.Projection.Boundary.AchievedLSN + resp.LastBarrierOK = snap.Projection.Boundary.LastBarrierOK + resp.LastBarrierReason = snap.Projection.Boundary.LastBarrierReason + } + if snap.HasCoreState { + resp.RecoveryPhase = string(snap.CoreState.Recovery.Phase) + if snap.CoreState.Boundary.DurableLSN > resp.DurableLSN { + resp.DurableLSN = snap.CoreState.Boundary.DurableLSN + } + if snap.CoreState.Boundary.TargetLSN > resp.TargetLSN { + resp.TargetLSN = snap.CoreState.Boundary.TargetLSN + } + if snap.CoreState.Boundary.AchievedLSN > resp.AchievedLSN { + resp.AchievedLSN = snap.CoreState.Boundary.AchievedLSN + } + if snap.CoreState.Boundary.CheckpointLSN > resp.CheckpointLSN { + resp.CheckpointLSN = snap.CoreState.Boundary.CheckpointLSN + } + if snap.CoreState.Boundary.LastBarrierReason != "" { + resp.LastBarrierReason = snap.CoreState.Boundary.LastBarrierReason + } + resp.LastBarrierOK = snap.CoreState.Boundary.LastBarrierOK + } + + switch { + case req.ExpectedEpoch != 0 && resp.Epoch != req.ExpectedEpoch: + resp.Reason = "epoch_mismatch" + case !resp.RoleApplied: + resp.Reason = "role_not_applied" + case resp.Mode == "needs_rebuild": + resp.Reason = "needs_rebuild" + default: + resp.Eligible = true + } + + return resp, nil +} + +// WriteLBA writes data to one named local volume. +func (n *Node) WriteLBA(name string, lba uint64, data []byte) error { + path, err := n.pathFor(name) + if err != nil { + return err + } + return n.dataPlane.WriteLBA(path, lba, data) +} + +// ReadLBA reads data from one named local volume. +func (n *Node) ReadLBA(name string, lba uint64, length uint32) ([]byte, error) { + path, err := n.pathFor(name) + if err != nil { + return nil, err + } + return n.dataPlane.ReadLBA(path, lba, length) +} + +// SyncCache flushes one named local volume. +func (n *Node) SyncCache(name string) error { + path, err := n.pathFor(name) + if err != nil { + return err + } + return n.dataPlane.SyncCache(path) +} + +// Snapshot returns the local debug snapshot for one named volume. +func (n *Node) Snapshot(name string) (purev2.VolumeDebugSnapshot, error) { + path, err := n.pathFor(name) + if err != nil { + return purev2.VolumeDebugSnapshot{}, err + } + return n.dataPlane.Snapshot(path) +} + +// Close shuts down the underlying pure runtime. +func (n *Node) Close() { + if n == nil || n.dataPlane == nil { + return + } + n.dataPlane.Close() +} + +func (n *Node) pathFor(name string) (string, error) { + n.mu.RLock() + defer n.mu.RUnlock() + binding, ok := n.volumes[name] + if !ok { + return "", fmt.Errorf("volumev2: unknown volume %q", name) + } + return binding.path, nil +}