feat: Phase 20 acceptance fixes + sw-test-runner suite mode

Acceptance rows closed:
- WriteLBA/SyncCache contract: code comments document write-back vs
  durability fence semantics
- RF=2 stable identity: v2bridge always uses SetReplicaAddrs (preserves
  ServerID); blockcmd dispatcher also fixed to use setupPrimaryReplicationMulti;
  test asserts exact expected ReplicaID="vs-2" (not just non-empty)
- Tests treating WriteLBA as commit: replica_read_test rewritten with
  SyncCache as durability fence
- publish_healthy contract: 3 gate tests with hard assertions including
  gate 3 (PrimaryShipperConnected)
- SetReplicaAddr deprecation warning added
- WALShipper.ReplicaID() getter added for identity verification

Test runner enhancements:
- sw-test-runner suite command: build → deploy → run N scenarios in one
  invocation with --skip-deploy support
- Suite YAML definitions for T6 Stage 0 and Stage 1
- deploy action: kill stale processes, clean dirs, cross-compile, upload
- run-phase20-t6.ps1 PowerShell script (deprecated by suite command)

Engine/runtime fixes:
- Recovery executor nil-safety improvements
- Recovery bundle BuildRecoveryBundle defensive checks
- ShipperGroup MinReplicaFlushedLSNAll surface

Docs: acceptance checklist refined, test matrix updated, T6 runbook,
engine maintainer tutorial, design README updated.

26 files changed, ~1600 insertions.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pingqiuandClaude Opus 4.6 committed 2026-04-06 11:30:54 -07:00
1 parent 275c3ee1c7
commit 44103a1bd7
27 files changed
+2472 -166

No files matched your search

@@ -1,14 +1,27 @@
package runtime
import engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
import (
"errors"
"strings"
engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
)
// RecoveryCallbacks is the host-side callback interface for recovery execution.
// The runtime helper drives plan execution; the host supplies concrete
// IO bindings and receives completion notifications.
type RecoveryCallbacks interface {
// OnRecoveryProgress is called when replay/rebuild reaches an explicit
// achieved boundary but the recovery session is not yet fully closed.
OnRecoveryProgress(volumeID, replicaID string, achievedLSN uint64)
// OnCatchUpCompleted is called after successful catch-up execution.
OnCatchUpCompleted(volumeID, replicaID string, achievedLSN uint64)
// OnCatchUpFailed is called when catch-up execution fails with a
// classified reason that the host may need to surface into core events.
OnCatchUpFailed(volumeID, replicaID, reason string)
// OnRebuildCompleted is called after successful rebuild execution.
// The host should read the post-rebuild snapshot and emit the
// appropriate core event.
@@ -28,6 +41,9 @@ func ExecuteCatchUpPlan(
exec := engine.NewCatchUpExecutor(driver, plan)
exec.IO = io
if err := exec.Execute(nil, 0); err != nil {
if callbacks != nil {
callbacks.OnCatchUpFailed(volumeID, replicaID, classifyCatchUpFailure(err))
}
return err
}
if callbacks != nil {
@@ -35,6 +51,7 @@ func ExecuteCatchUpPlan(
if achievedLSN == 0 {
achievedLSN = plan.CatchUpStartLSN
}
callbacks.OnRecoveryProgress(volumeID, replicaID, achievedLSN)
callbacks.OnCatchUpCompleted(volumeID, replicaID, achievedLSN)
}
return nil
@@ -56,7 +73,31 @@ func ExecuteRebuildPlan(
return err
}
if callbacks != nil {
callbacks.OnRecoveryProgress(volumeID, replicaID, plan.RebuildTargetLSN)
callbacks.OnRebuildCompleted(volumeID, replicaID, plan)
}
return nil
}
func classifyCatchUpFailure(err error) string {
if err == nil {
return ""
}
msg := err.Error()
switch {
case errors.Is(err, engine.ErrTruncationUnsafe):
return "truncation_unsafe"
case strings.Contains(msg, "WAL recycled"):
return "retention_lost"
case strings.Contains(msg, "duration_exceeded"):
return "catchup_duration_exceeded"
case strings.Contains(msg, "progress_stalled"):
return "catchup_progress_stalled"
case strings.Contains(msg, "entries_limit_exceeded"):
return "catchup_entries_limit_exceeded"
case strings.Contains(msg, "budget violation"):
return "catchup_budget_exceeded"
default:
return ""
}
}
@@ -1,12 +1,17 @@
package runtime
import (
"errors"
"fmt"
"testing"
engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
)
type fakeCallbacks struct {
progressCalled bool
progressLSN uint64
catchUpCalled bool
catchUpVol string
catchUpReplica string
@@ -16,6 +21,14 @@ type fakeCallbacks struct {
rebuildVol string
rebuildReplica string
rebuildPlan *engine.RecoveryPlan
catchUpFailedCalled bool
catchUpFailedReason string
}
func (f *fakeCallbacks) OnRecoveryProgress(volumeID, replicaID string, achievedLSN uint64) {
f.progressCalled = true
f.progressLSN = achievedLSN
}
func (f *fakeCallbacks) OnCatchUpCompleted(volumeID, replicaID string, achievedLSN uint64) {
@@ -25,6 +38,11 @@ func (f *fakeCallbacks) OnCatchUpCompleted(volumeID, replicaID string, achievedL
f.catchUpLSN = achievedLSN
}
func (f *fakeCallbacks) OnCatchUpFailed(volumeID, replicaID, reason string) {
f.catchUpFailedCalled = true
f.catchUpFailedReason = reason
}
func (f *fakeCallbacks) OnRebuildCompleted(volumeID, replicaID string, plan *engine.RecoveryPlan) {
f.rebuildCalled = true
f.rebuildVol = volumeID
@@ -61,6 +79,9 @@ func TestExecuteCatchUpPlan_CallsbackOnSuccess(t *testing.T) {
if !cb.catchUpCalled {
t.Fatal("callback not called")
}
if !cb.progressCalled || cb.progressLSN != cb.catchUpLSN {
t.Fatalf("progress callback mismatch: called=%v progress=%d catchup=%d", cb.progressCalled, cb.progressLSN, cb.catchUpLSN)
}
if cb.catchUpVol != "vol1" {
t.Fatalf("vol=%s", cb.catchUpVol)
}
@@ -137,6 +158,56 @@ func TestExecuteCatchUpPlan_NilCallbacksSafe(t *testing.T) {
}
}
func TestExecuteCatchUpPlan_CallsbackOnFailureWithClassification(t *testing.T) {
cb := &fakeCallbacks{}
driver := setupDriver(t, "vol1/vs2")
plan, err := driver.PlanRecovery("vol1/vs2", 50)
if err != nil {
t.Fatal(err)
}
err = ExecuteCatchUpPlan(driver, plan, failingCatchUpIO{err: errors.New("WAL recycled before catch-up could complete")}, "vol1", "vol1/vs2", cb)
if err == nil {
t.Fatal("expected catch-up failure")
}
if !cb.catchUpFailedCalled {
t.Fatal("expected failure callback")
}
if cb.catchUpFailedReason != "retention_lost" {
t.Fatalf("failure reason=%q, want retention_lost", cb.catchUpFailedReason)
}
if cb.progressCalled {
t.Fatal("progress callback should not fire on failure")
}
if cb.catchUpCalled {
t.Fatal("completion callback should not fire on failure")
}
}
func TestClassifyCatchUpFailure(t *testing.T) {
tests := []struct {
name string
err error
want string
}{
{name: "nil", err: nil, want: ""},
{name: "truncation unsafe", err: fmt.Errorf("wrapped: %w", engine.ErrTruncationUnsafe), want: "truncation_unsafe"},
{name: "retention lost", err: errors.New("WAL recycled while replaying"), want: "retention_lost"},
{name: "duration exceeded", err: errors.New("duration_exceeded"), want: "catchup_duration_exceeded"},
{name: "progress stalled", err: errors.New("progress_stalled"), want: "catchup_progress_stalled"},
{name: "entries limit", err: errors.New("entries_limit_exceeded"), want: "catchup_entries_limit_exceeded"},
{name: "budget exceeded", err: errors.New("budget violation"), want: "catchup_budget_exceeded"},
{name: "unknown", err: errors.New("boom"), want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := classifyCatchUpFailure(tt.err); got != tt.want {
t.Fatalf("classifyCatchUpFailure(%v)=%q, want %q", tt.err, got, tt.want)
}
})
}
}
// --- test helpers ---
type noopCatchUpIO struct{}
@@ -144,6 +215,11 @@ type noopCatchUpIO struct{}
func (noopCatchUpIO) StreamWALEntries(start, end uint64) (uint64, error) { return end, nil }
func (noopCatchUpIO) TruncateWAL(lsn uint64) error { return nil }
type failingCatchUpIO struct{ err error }
func (f failingCatchUpIO) StreamWALEntries(start, end uint64) (uint64, error) { return 0, f.err }
func (f failingCatchUpIO) TruncateWAL(lsn uint64) error { return nil }
type noopRebuildIO struct{}
func (noopRebuildIO) StreamWALEntries(start, end uint64) (uint64, error) { return end, nil }