seaweedfs/weed/storage/blockvol/sync_all_adversarial_test.go

package blockvol

// CP13-5 adversarial tests: edge cases for reconnect, catch-up, and state machine.
// These test the 6 audit points from the CP13-5 review.

import (
	"bytes"
	"sync"
	"testing"
	"time"
)

// ---------- Point 1: catchupFailures concurrency ----------

// TestAdversarial_ConcurrentBarrierDoesNotCorruptCatchupFailures verifies
// that rapid concurrent SyncCache calls (which trigger Barrier on the same
// shipper) do not corrupt the catchupFailures counter.
// The group committer serializes SyncCache, but this test exercises the
// boundary by calling Barrier directly from multiple goroutines.
func TestAdversarial_ConcurrentBarrierDoesNotCorruptCatchupFailures(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()
	defer recv.Stop()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Write + sync to establish InSync.
	if err := primary.WriteLBA(0, makeBlock('A')); err != nil {
		t.Fatal(err)
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Fire 10 concurrent SyncCache calls.
	var wg sync.WaitGroup
	errors := make([]error, 10)
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			if err := primary.WriteLBA(uint64(idx+1), makeBlock(byte('B'+idx))); err != nil {
				errors[idx] = err
				return
			}
			errors[idx] = primary.SyncCache()
		}(i)
	}
	wg.Wait()

	// All should succeed (healthy path).
	for i, err := range errors {
		if err != nil {
			t.Errorf("concurrent SyncCache[%d]: %v", i, err)
		}
	}
}

// ---------- Point 2: bootstrap vs reconnect discriminator ----------

// TestAdversarial_FreshShipperUsesBootstrapNotReconnect verifies that a
// freshly created shipper (hasFlushedProgress=false) uses the bootstrap
// path (bare TCP connect), not the reconnect handshake path.
func TestAdversarial_FreshShipperUsesBootstrapNotReconnect(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()
	defer recv.Stop()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	sg := primary.shipperGroup
	s := sg.Shipper(0)
	if s == nil {
		t.Fatal("no shipper")
	}

	// Fresh shipper: hasFlushedProgress must be false.
	if s.HasFlushedProgress() {
		t.Fatal("fresh shipper should not have flushed progress")
	}

	// State should be Disconnected (initial).
	if s.State() != ReplicaDisconnected {
		t.Fatalf("fresh shipper state=%s, want Disconnected", s.State())
	}

	// First write + sync should succeed via bootstrap path.
	if err := primary.WriteLBA(0, makeBlock('X')); err != nil {
		t.Fatal(err)
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatalf("first SyncCache (bootstrap): %v", err)
	}

	// After first successful barrier, hasFlushedProgress should be true.
	if !s.HasFlushedProgress() {
		t.Fatal("after successful barrier, hasFlushedProgress should be true")
	}
	if s.State() != ReplicaInSync {
		t.Fatalf("after bootstrap barrier, state=%s, want InSync", s.State())
	}
}

// TestAdversarial_ReconnectUsesHandshakeNotBootstrap verifies that after
// a degraded shipper reconnects, it uses the handshake protocol (not bare
// TCP retry) because hasFlushedProgress is true.
func TestAdversarial_ReconnectUsesHandshakeNotBootstrap(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()
	defer recv.Stop()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Establish InSync.
	if err := primary.WriteLBA(0, makeBlock('A')); err != nil {
		t.Fatal(err)
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	sg := primary.shipperGroup
	s := sg.Shipper(0)
	if !s.HasFlushedProgress() {
		t.Fatal("should have flushed progress after sync")
	}

	// Disconnect replica.
	recv.Stop()
	time.Sleep(50 * time.Millisecond)

	// Write during disconnect.
	if err := primary.WriteLBA(1, makeBlock('B')); err != nil {
		t.Fatal(err)
	}

	// Reconnect.
	recv2, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv2.Serve()
	defer recv2.Stop()

	// Reconfigure shipper to new address.
	// CP13-5: SetReplicaAddrs creates fresh shippers but seeds them with
	// hasFlushedProgress=true from the old group, so the new shipper uses
	// the reconnect handshake (ResumeShipReq) path, not bare bootstrap.
	primary.SetReplicaAddr(recv2.DataAddr(), recv2.CtrlAddr())

	// CP13-5 observable evidence 1: new shipper seeded with prior progress.
	newSg := primary.shipperGroup
	newS := newSg.Shipper(0)
	if !newS.HasFlushedProgress() {
		t.Fatal("CP13-5: new shipper should be seeded with hasFlushedProgress=true from old group")
	}

	// Record replica's receivedLSN before SyncCache to prove catch-up delivers entries.
	preRecvLSN := recv2.ReceivedLSN()

	syncDone := make(chan error, 1)
	go func() {
		syncDone <- primary.SyncCache()
	}()

	select {
	case err := <-syncDone:
		if err != nil {
			t.Fatalf("SyncCache after reconnect: %v", err)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("SyncCache hung after reconnect")
	}

	// CP13-5 observable evidence 2: replica received entries via catch-up.
	// Block 'B' (LSN 2) was written during disconnect. If the handshake +
	// catch-up path was used, the replica's receivedLSN must have advanced.
	// Bootstrap alone would not deliver block 'B' — it only sends the barrier.
	postRecvLSN := recv2.ReceivedLSN()
	if postRecvLSN <= preRecvLSN {
		t.Fatalf("CP13-5: replica receivedLSN did not advance (%d → %d) — catch-up did not deliver entries",
			preRecvLSN, postRecvLSN)
	}

	// CP13-5 observable evidence 3: shipper now has updated flushedLSN from barrier.
	if newS.ReplicaFlushedLSN() == 0 {
		t.Fatal("CP13-5: shipper should have replicaFlushedLSN > 0 after successful barrier")
	}
}

// ---------- Point 3: duplicate catch-up LSN semantics ----------

// TestAdversarial_ReplicaRejectsDuplicateLSN verifies the replica skips
// entries with LSN <= receivedLSN (duplicate/old), does not error.
func TestAdversarial_ReplicaRejectsDuplicateLSN(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()
	defer recv.Stop()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Write 5 entries.
	for i := uint64(0); i < 5; i++ {
		if err := primary.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatal(err)
		}
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Verify replica has all 5.
	if recv.ReceivedLSN() < 5 {
		t.Fatalf("replica receivedLSN=%d, expected >=5", recv.ReceivedLSN())
	}

	// Manually send a duplicate entry (LSN 3) to the replica.
	// This should be silently skipped, not error.
	entry := &WALEntry{
		LSN:    3, // already received
		Epoch:  1,
		Type:   EntryTypeWrite,
		LBA:    100,
		Length: 4096,
		Data:   makeBlock('Z'),
	}
	err = recv.ApplyEntryForTest(entry)
	if err != nil {
		t.Fatalf("duplicate LSN should be skipped, got error: %v", err)
	}

	// Original data at LBA 2 (LSN 3) should be unchanged.
	replica.flusher.FlushOnce()
	got, _ := replica.ReadLBA(2, 4096)
	if got[0] != 'C' {
		t.Fatalf("LBA 2: expected C, got %c — duplicate entry corrupted data", got[0])
	}
}

// TestAdversarial_ReplicaRejectsGapLSN verifies the replica rejects entries
// with LSN > receivedLSN+1 (gap — entries were missed).
func TestAdversarial_ReplicaRejectsGapLSN(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()
	defer recv.Stop()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Write 3 entries.
	for i := uint64(0); i < 3; i++ {
		if err := primary.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatal(err)
		}
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Manually send LSN 10 (skipping 4-9). Should fail with gap error.
	entry := &WALEntry{
		LSN:    10,
		Epoch:  1,
		Type:   EntryTypeWrite,
		LBA:    50,
		Length: 4096,
		Data:   makeBlock('Z'),
	}
	err = recv.ApplyEntryForTest(entry)
	if err == nil {
		t.Fatal("gap LSN should be rejected, got nil error")
	}
}

// ---------- Point 4: NeedsRebuild stickiness ----------

// TestAdversarial_NeedsRebuildBlocksAllPaths verifies that once a shipper
// enters NeedsRebuild, neither Ship nor Barrier can bring it back to healthy.
//
// CP13-7 proof: NeedsRebuild is a sticky fail-closed state.
func TestAdversarial_NeedsRebuildBlocksAllPaths(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Establish sync.
	if err := primary.WriteLBA(0, makeBlock('A')); err != nil {
		t.Fatal(err)
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	sg := primary.shipperGroup
	s := sg.Shipper(0)
	if s.State() != ReplicaInSync {
		t.Fatalf("expected InSync, got %s", s.State())
	}

	// Disconnect replica and write a few entries (within WAL capacity).
	recv.Stop()
	time.Sleep(50 * time.Millisecond)

	for i := uint64(1); i < 8; i++ {
		if err := primary.WriteLBA(i, makeBlock(byte('0'+i))); err != nil {
			t.Fatalf("write %d: %v", i, err)
		}
	}

	// CP13-6 → CP13-7: trigger NeedsRebuild via max-bytes budget (small threshold).
	sg.EvaluateRetentionBudgets(RetentionBudgetParams{
		Timeout:        5 * time.Minute,
		MaxBytes:       4 * 1024, // 4KB — lag of 7 entries * 4KB = 28KB > 4KB
		PrimaryHeadLSN: primary.nextLSN.Load() - 1,
		BlockSize:      primary.super.BlockSize,
	})

	// 1. NeedsRebuild state asserted.
	st := s.State()
	if st != ReplicaNeedsRebuild {
		t.Fatalf("CP13-7: expected NeedsRebuild after budget exceeded, got %s", st)
	}

	// 2. Ship must silently drop — NeedsRebuild shipper must not participate.
	if err := primary.WriteLBA(0, makeBlock('Z')); err != nil {
		t.Fatalf("write should succeed (local WAL): %v", err)
	}
	st2 := s.State()
	if st2 != ReplicaNeedsRebuild {
		t.Fatalf("CP13-7: Ship should not change NeedsRebuild state, got %s", st2)
	}

	// 3. Barrier/SyncCache must fail — NeedsRebuild rejected by Barrier() state gate.
	syncDone := make(chan error, 1)
	go func() {
		syncDone <- primary.SyncCache()
	}()
	select {
	case err := <-syncDone:
		if err == nil {
			t.Fatal("CP13-7: SyncCache should fail with NeedsRebuild shipper")
		}
	case <-time.After(10 * time.Second):
		t.Fatal("SyncCache hung")
	}

	// 4. State is still NeedsRebuild after failed barrier (sticky).
	st3 := s.State()
	if st3 != ReplicaNeedsRebuild {
		t.Fatalf("CP13-7: NeedsRebuild should be sticky after failed barrier, got %s", st3)
	}

	// 5. Second SyncCache also fails (not recovered by retry).
	syncDone2 := make(chan error, 1)
	go func() {
		syncDone2 <- primary.SyncCache()
	}()
	select {
	case err := <-syncDone2:
		if err == nil {
			t.Fatal("CP13-7: second SyncCache should still fail")
		}
	case <-time.After(10 * time.Second):
		t.Fatal("second SyncCache hung")
	}

	t.Log("CP13-7: NeedsRebuild is fail-closed — Ship drops, Barrier rejects, state sticky")
}

// ---------- Point 6: data integrity after catch-up ----------

// TestAdversarial_CatchupDoesNotOverwriteNewerData verifies that if the
// replica has data at an LBA from a later LSN, catch-up replay of an
// earlier LSN for the same LBA does not overwrite the newer version.
// (This is actually handled by the WAL: the dirty map always uses the
// latest LSN for each LBA.)
func TestAdversarial_CatchupDoesNotOverwriteNewerData(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()
	defer recv.Stop()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Write LBA 0 = 'A' (LSN 1), then LBA 0 = 'B' (LSN 2).
	if err := primary.WriteLBA(0, makeBlock('A')); err != nil {
		t.Fatal(err)
	}
	if err := primary.WriteLBA(0, makeBlock('B')); err != nil {
		t.Fatal(err)
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Disconnect, write LBA 0 = 'C' (LSN 3).
	recv.Stop()
	time.Sleep(50 * time.Millisecond)

	if err := primary.WriteLBA(0, makeBlock('C')); err != nil {
		t.Fatal(err)
	}

	// Reconnect — catch-up sends LSN 3.
	recv2, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv2.Serve()
	defer recv2.Stop()
	primary.SetReplicaAddr(recv2.DataAddr(), recv2.CtrlAddr())

	syncDone := make(chan error, 1)
	go func() {
		syncDone <- primary.SyncCache()
	}()

	select {
	case err := <-syncDone:
		if err != nil {
			t.Fatalf("SyncCache: %v", err)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("SyncCache hung")
	}

	// Replica should have 'C' at LBA 0, not 'A' or 'B'.
	replica.flusher.FlushOnce()
	got, err := replica.ReadLBA(0, 4096)
	if err != nil {
		t.Fatal(err)
	}
	if got[0] != 'C' {
		t.Fatalf("LBA 0: expected C (latest), got %c — catch-up overwrote newer data", got[0])
	}
}

// TestAdversarial_CatchupMultipleDisconnects verifies that multiple
// disconnect/reconnect cycles with writes in between all converge correctly.
func TestAdversarial_CatchupMultipleDisconnects(t *testing.T) {
	primary, replica := createSyncAllPair(t)
	defer primary.Close()
	defer replica.Close()

	recv, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv.Serve()

	primary.SetReplicaAddr(recv.DataAddr(), recv.CtrlAddr())

	// Cycle 1: write, sync, disconnect, write.
	for i := uint64(0); i < 3; i++ {
		if err := primary.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatal(err)
		}
	}
	if err := primary.SyncCache(); err != nil {
		t.Fatal(err)
	}

	recv.Stop()
	time.Sleep(30 * time.Millisecond)

	for i := uint64(3); i < 5; i++ {
		if err := primary.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatal(err)
		}
	}

	// Reconnect 1.
	recv2, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv2.Serve()
	primary.SetReplicaAddr(recv2.DataAddr(), recv2.CtrlAddr())

	if err := primary.SyncCache(); err != nil {
		t.Fatalf("cycle 1 reconnect SyncCache: %v", err)
	}

	// Cycle 2: disconnect again, write more.
	recv2.Stop()
	time.Sleep(30 * time.Millisecond)

	for i := uint64(5); i < 8; i++ {
		if err := primary.WriteLBA(i, makeBlock(byte('A'+i))); err != nil {
			t.Fatal(err)
		}
	}

	// Reconnect 2.
	recv3, err := NewReplicaReceiver(replica, "127.0.0.1:0", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	recv3.Serve()
	defer recv3.Stop()
	primary.SetReplicaAddr(recv3.DataAddr(), recv3.CtrlAddr())

	if err := primary.SyncCache(); err != nil {
		t.Fatalf("cycle 2 reconnect SyncCache: %v", err)
	}

	// Verify all 8 blocks on replica.
	replica.flusher.FlushOnce()
	for i := uint64(0); i < 8; i++ {
		got, err := replica.ReadLBA(i, 4096)
		if err != nil {
			t.Fatalf("ReadLBA(%d): %v", i, err)
		}
		expected := byte('A' + i)
		if !bytes.Equal(got[:1], []byte{expected}) {
			t.Errorf("LBA %d: expected %c, got %c after 2 disconnect/reconnect cycles", i, expected, got[0])
		}
	}
}