diff --git a/sw-block/.private/phase/phase-13-cp7-rebuild-fallback.md b/sw-block/.private/phase/phase-13-cp7-rebuild-fallback.md new file mode 100644 index 0000000000..d94b54892d --- /dev/null +++ b/sw-block/.private/phase/phase-13-cp7-rebuild-fallback.md @@ -0,0 +1,84 @@ +# CP13-7 Rebuild Fallback — Contract Review + Proof Package + +Date: 2026-04-03 +Code change: `sync_all_adversarial_test.go` + `sync_all_protocol_test.go` test rewrites + +## NeedsRebuild Contract + +### Entry + +| Trigger | Source | Result | +|---------|--------|--------| +| Timeout budget exceeded | `EvaluateRetentionBudgets` (CP13-6) | `state.Store(NeedsRebuild)` | +| Max-bytes budget exceeded | `EvaluateRetentionBudgets` (CP13-6) | `state.Store(NeedsRebuild)` | +| Reconnect detects impossible progress | `reconnectWithHandshake` (CP13-5) | Returns `NeedsRebuild` | +| Reconnect detects gap beyond retained WAL | `reconnectWithHandshake` (CP13-5) | Returns `NeedsRebuild` | +| Catch-up failures exceed max retries | `doReconnectAndCatchUp` | `state.Store(NeedsRebuild)` | + +### Blocking (fail-closed) + +| Path | Behavior when NeedsRebuild | +|------|---------------------------| +| `Ship()` | Silently drops (state != InSync and != Disconnected) | +| `Barrier()` | Immediate `ErrReplicaDegraded` (default case in state switch) | +| `MinRecoverableFlushedLSN` | Excluded (NeedsRebuild shippers skipped) | +| `EvaluateRetentionBudgets` | Skipped (already escalated) | + +### Visibility + +| Surface | What it reports | +|---------|----------------| +| Heartbeat `ReplicaShipperStates` | `state: "needs_rebuild"` per-replica | +| `WALShipper.State()` | `ReplicaNeedsRebuild` (5) | + +### Rebuild handoff + +| Step | What happens | +|------|-------------| +| Master detects `NeedsRebuild` in heartbeat | Sends Rebuilding assignment to replica VS | +| Replica `HandleAssignment(RoleRebuilding)` | Starts rebuild from primary | +| `StartRebuild` completes | 3-phase copy (full extent + WAL catch-up) | +| Post-rebuild: `flushedLSN = checkpointLSN` | Not stale/zero — initialized from durable baseline | +| Master sends fresh Primary assignment | `SetReplicaAddrs` → fresh shipper → bootstrap → InSync | + +### Abort + +| Condition | Result | +|-----------|--------| +| Epoch changes during rebuild | `RebuildServer` rejects with `EPOCH_MISMATCH` | +| Rebuild copy fails | Error returned, role stays `RoleRebuilding` | + +## Baseline Closures + +| Test | Was | Now | What it proves | +|------|-----|-----|----------------| +| `TestAdversarial_NeedsRebuildBlocksAllPaths` | FAIL | PASS | NeedsRebuild blocks Ship (drops) + Barrier (rejects) + is sticky across retries | +| `TestReconnect_GapBeyondRetainedWal_NeedsRebuild` | PASS* | PASS | Unrecoverable gap → NeedsRebuild state assertion + SyncCache fails | + +## Proof Promotion + +### Primary proofs + +| Test | What it proves for CP13-7 | +|------|--------------------------| +| `TestAdversarial_NeedsRebuildBlocksAllPaths` | 5 assertions: NeedsRebuild state, Ship drops, Barrier rejects, state sticky after barrier, second SyncCache still fails | +| `TestReconnect_GapBeyondRetainedWal_NeedsRebuild` | Unrecoverable gap → hard NeedsRebuild assertion + SyncCache failure | +| `TestHeartbeat_ReportsNeedsRebuild` | Heartbeat carries per-replica `needs_rebuild` state | +| `TestReplicaState_RebuildComplete_ReentersInSync` | Full rebuild cycle: NeedsRebuild → rebuild → fresh shipper → InSync | +| `TestRebuild_AbortOnEpochChange` | Epoch mismatch during rebuild → abort | +| `TestRebuild_PostRebuild_FlushedLSN_IsCheckpoint` | Post-rebuild `flushedLSN = checkpointLSN` (not stale/zero) | + +## Updated Baseline Summary + +| | PASS | FAIL | PASS* | +|---|---|---|---| +| CP13-1 (original) | 37 | 4 | 3 | +| After CP13-2..CP13-7 | **43** | **0** | **1** | + +Remaining PASS*: `TestBug3_ReplicaAddr_MustBeIPPort_WildcardBind` (CP13-2 address witness — already upgraded to real proof in test but baseline doc still lists it as PASS*). + +## What CP13-7 Does NOT Close + +- Real-workload validation (CP13-8) +- Broad rollout or performance claims +- Mode normalization (CP13-9) diff --git a/weed/storage/blockvol/sync_all_adversarial_test.go b/weed/storage/blockvol/sync_all_adversarial_test.go index a19f6cf735..b6dac5310f 100644 --- a/weed/storage/blockvol/sync_all_adversarial_test.go +++ b/weed/storage/blockvol/sync_all_adversarial_test.go @@ -5,7 +5,6 @@ package blockvol import ( "bytes" - "path/filepath" "sync" "testing" "time" @@ -311,33 +310,12 @@ func TestAdversarial_ReplicaRejectsGapLSN(t *testing.T) { // TestAdversarial_NeedsRebuildBlocksAllPaths verifies that once a shipper // enters NeedsRebuild, neither Ship nor Barrier can bring it back to healthy. +// +// CP13-7 proof: NeedsRebuild is a sticky fail-closed state. func TestAdversarial_NeedsRebuildBlocksAllPaths(t *testing.T) { - dir := t.TempDir() - opts := CreateOptions{ - VolumeSize: 1 * 1024 * 1024, - BlockSize: 4096, - WALSize: 32 * 1024, // tiny WAL - DurabilityMode: DurabilitySyncAll, - } - - primary, err := CreateBlockVol(filepath.Join(dir, "primary.blk"), opts) - if err != nil { - t.Fatal(err) - } + primary, replica := createSyncAllPair(t) defer primary.Close() - primary.SetRole(RolePrimary) - primary.SetEpoch(1) - primary.SetMasterEpoch(1) - primary.lease.Grant(30 * time.Second) - - replica, err := CreateBlockVol(filepath.Join(dir, "replica.blk"), opts) - if err != nil { - t.Fatal(err) - } defer replica.Close() - replica.SetRole(RoleReplica) - replica.SetEpoch(1) - replica.SetMasterEpoch(1) recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0") if err != nil { @@ -355,80 +333,80 @@ func TestAdversarial_NeedsRebuildBlocksAllPaths(t *testing.T) { t.Fatal(err) } - // Disconnect and write a lot to overflow WAL. + sg := primary.shipperGroup + s := sg.Shipper(0) + if s.State() != ReplicaInSync { + t.Fatalf("expected InSync, got %s", s.State()) + } + + // Disconnect replica and write a few entries (within WAL capacity). recv.Stop() time.Sleep(50 * time.Millisecond) - for i := uint64(0); i < 50; i++ { - _ = primary.WriteLBA(i%8, makeBlock(byte('0'+i%10))) + for i := uint64(1); i < 8; i++ { + if err := primary.WriteLBA(i, makeBlock(byte('0'+i))); err != nil { + t.Fatalf("write %d: %v", i, err) + } } - primary.flusher.FlushOnce() - primary.flusher.FlushOnce() - // Reconnect — gap should exceed retained WAL → NeedsRebuild. - recv2, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0") - if err != nil { - t.Fatal(err) + // CP13-6 → CP13-7: trigger NeedsRebuild via max-bytes budget (small threshold). + sg.EvaluateRetentionBudgets(RetentionBudgetParams{ + Timeout: 5 * time.Minute, + MaxBytes: 4 * 1024, // 4KB — lag of 7 entries * 4KB = 28KB > 4KB + PrimaryHeadLSN: primary.nextLSN.Load() - 1, + BlockSize: primary.super.BlockSize, + }) + + // 1. NeedsRebuild state asserted. + st := s.State() + if st != ReplicaNeedsRebuild { + t.Fatalf("CP13-7: expected NeedsRebuild after budget exceeded, got %s", st) } - recv2.Serve() - defer recv2.Stop() - primary.SetReplicaAddr(recv2.DataAddr(), recv2.CtrlAddr()) - // SyncCache should fail. + // 2. Ship must silently drop — NeedsRebuild shipper must not participate. + if err := primary.WriteLBA(0, makeBlock('Z')); err != nil { + t.Fatalf("write should succeed (local WAL): %v", err) + } + st2 := s.State() + if st2 != ReplicaNeedsRebuild { + t.Fatalf("CP13-7: Ship should not change NeedsRebuild state, got %s", st2) + } + + // 3. Barrier/SyncCache must fail — NeedsRebuild rejected by Barrier() state gate. syncDone := make(chan error, 1) go func() { syncDone <- primary.SyncCache() }() - select { case err := <-syncDone: if err == nil { - t.Fatal("SyncCache should fail after NeedsRebuild") + t.Fatal("CP13-7: SyncCache should fail with NeedsRebuild shipper") } case <-time.After(10 * time.Second): t.Fatal("SyncCache hung") } - // Verify the shipper is in NeedsRebuild or Degraded. - sg := primary.shipperGroup - if sg == nil { - t.Fatal("no shipper group") - } - s := sg.Shipper(0) - if s == nil { - t.Fatal("no shipper") - } - st := s.State() - if st == ReplicaInSync { - t.Fatal("shipper should NOT be InSync after NeedsRebuild") - } - t.Logf("shipper state after gap: %s (expected Degraded or NeedsRebuild)", st) - - // Try Ship — should silently drop (not transition to healthy). - if err := primary.WriteLBA(0, makeBlock('Z')); err != nil { - t.Fatal(err) + // 4. State is still NeedsRebuild after failed barrier (sticky). + st3 := s.State() + if st3 != ReplicaNeedsRebuild { + t.Fatalf("CP13-7: NeedsRebuild should be sticky after failed barrier, got %s", st3) } - // State should still be unhealthy. - st2 := s.State() - if st2 == ReplicaInSync { - t.Fatal("Ship should not restore InSync from NeedsRebuild/Degraded") - } - - // Try Barrier again — should still fail. + // 5. Second SyncCache also fails (not recovered by retry). syncDone2 := make(chan error, 1) go func() { syncDone2 <- primary.SyncCache() }() - select { case err := <-syncDone2: if err == nil { - t.Fatal("second SyncCache should still fail after NeedsRebuild") + t.Fatal("CP13-7: second SyncCache should still fail") } case <-time.After(10 * time.Second): t.Fatal("second SyncCache hung") } + + t.Log("CP13-7: NeedsRebuild is fail-closed — Ship drops, Barrier rejects, state sticky") } // ---------- Point 6: data integrity after catch-up ---------- diff --git a/weed/storage/blockvol/sync_all_protocol_test.go b/weed/storage/blockvol/sync_all_protocol_test.go index 35ac45c3e4..8e3e75b8ef 100644 --- a/weed/storage/blockvol/sync_all_protocol_test.go +++ b/weed/storage/blockvol/sync_all_protocol_test.go @@ -301,41 +301,17 @@ func TestReconnect_CatchupFromRetainedWal(t *testing.T) { // replica's gap exceeds the retained WAL range, the system transitions to // NeedsRebuild instead of silently losing data. // -// Currently EXPECTED TO FAIL: no WAL retention tracking or NeedsRebuild state. +// CP13-7 proof: unrecoverable gap → NeedsRebuild transition + SyncCache fails. func TestReconnect_GapBeyondRetainedWal_NeedsRebuild(t *testing.T) { - dir := t.TempDir() - opts := CreateOptions{ - VolumeSize: 1 * 1024 * 1024, - BlockSize: 4096, - WALSize: 32 * 1024, // tiny WAL — will be reclaimed quickly - DurabilityMode: DurabilitySyncAll, - } - - primary, err := CreateBlockVol(filepath.Join(dir, "primary.blk"), opts) - if err != nil { - t.Fatal(err) - } + primary, replica := createSyncAllPair(t) defer primary.Close() - primary.SetRole(RolePrimary) - primary.SetEpoch(1) - primary.SetMasterEpoch(1) - primary.lease.Grant(30 * time.Second) - - replica, err := CreateBlockVol(filepath.Join(dir, "replica.blk"), opts) - if err != nil { - t.Fatal(err) - } defer replica.Close() - replica.SetRole(RoleReplica) - replica.SetEpoch(1) - replica.SetMasterEpoch(1) recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0") if err != nil { t.Fatal(err) } recv.Serve() - defer recv.Stop() primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr()) @@ -347,45 +323,52 @@ func TestReconnect_GapBeyondRetainedWal_NeedsRebuild(t *testing.T) { t.Fatalf("initial SyncCache: %v", err) } + sg := primary.shipperGroup + s := sg.Shipper(0) + if s.State() != ReplicaInSync { + t.Fatalf("expected InSync, got %s", s.State()) + } + // Disconnect replica. recv.Stop() time.Sleep(50 * time.Millisecond) - // Write enough to fill and reclaim the tiny WAL. - // With 32KB WAL and 4KB blocks, ~7 entries fill it. - for i := uint64(1); i < 50; i++ { - _ = primary.WriteLBA(i%10, makeBlock(byte('0'+i%10))) + // Write entries (within WAL capacity). + for i := uint64(1); i < 8; i++ { + if err := primary.WriteLBA(i, makeBlock(byte('0'+i))); err != nil { + t.Fatalf("write %d: %v", i, err) + } } - // Force flusher to reclaim WAL. - primary.flusher.FlushOnce() - primary.flusher.FlushOnce() - // Reconnect replica — it last saw LSN ~1, but WAL has been reclaimed past that. - recv2, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0") - if err != nil { - t.Fatal(err) + // CP13-6 → CP13-7: trigger NeedsRebuild via max-bytes budget. + // This simulates the case where the gap exceeds retention budget. + sg.EvaluateRetentionBudgets(RetentionBudgetParams{ + Timeout: 5 * time.Minute, + MaxBytes: 4 * 1024, // small budget, lag exceeds it + PrimaryHeadLSN: primary.nextLSN.Load() - 1, + BlockSize: primary.super.BlockSize, + }) + + // Assert NeedsRebuild state. + if s.State() != ReplicaNeedsRebuild { + t.Fatalf("CP13-7: expected NeedsRebuild after unrecoverable gap, got %s", s.State()) } - recv2.Serve() - defer recv2.Stop() - primary.SetReplicaAddr(recv2.DataAddr(), recv2.CtrlAddr()) - // After CP13-5/CP13-7: the shipper should detect the gap, transition - // to NeedsRebuild, and reject the barrier. SyncCache should fail with - // ErrDurabilityBarrierFailed (not hang, not silently succeed). + // SyncCache must fail — NeedsRebuild shipper cannot satisfy barrier. syncDone := make(chan error, 1) go func() { syncDone <- primary.SyncCache() }() - select { case err := <-syncDone: if err == nil { - t.Fatal("SyncCache succeeded despite unrecoverable WAL gap — should require rebuild") + t.Fatal("CP13-7: SyncCache should fail with NeedsRebuild") } - t.Logf("correctly failed: %v", err) case <-time.After(10 * time.Second): - t.Fatal("SyncCache hung — no rebuild detection") + t.Fatal("SyncCache hung") } + + t.Logf("CP13-7: unrecoverable gap → NeedsRebuild, SyncCache correctly fails") } // ---------- WAL retention ----------