seaweedfs/weed/server/qa_block_ha_edge_cases_test.go

package weed_server

import (
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)

// ============================================================
// HA Edge Case Tests (from edge-cases-failover.md)
// ============================================================

// --- EC-6: WAL LSN blocks promotion when primary dies under load ---

func TestEC6_WALLagBlocksPromotion(t *testing.T) {
	r := NewBlockVolumeRegistry()
	r.MarkBlockCapable("primary:9333")
	r.MarkBlockCapable("replica:9333")

	r.Register(&BlockVolumeEntry{
		Name:           "ec6-vol",
		VolumeServer:   "primary:9333",
		Path:           "/data/ec6-vol.blk",
		SizeBytes:      1 << 30,
		Epoch:          1,
		Role:           blockvol.RoleToWire(blockvol.RolePrimary),
		Status:         StatusActive,
		LeaseTTL:       30 * time.Second,
		LastLeaseGrant: time.Now(),
		WALHeadLSN:     500,
		Replicas: []ReplicaInfo{
			{
				Server:        "replica:9333",
				Path:          "/data/ec6-vol.blk",
				Role:          blockvol.RoleToWire(blockvol.RoleReplica),
				WALHeadLSN:    350, // 150 behind — exceeds tolerance of 100
				HealthScore:   1.0,
				LastHeartbeat: time.Now(),
			},
		},
	})

	// Primary dies.
	r.UnmarkBlockCapable("primary:9333")

	// Auto-promote should FAIL.
	_, err := r.PromoteBestReplica("ec6-vol")

	if err == nil {
		t.Log("EC-6 NOT reproduced: promotion succeeded despite 150-entry lag")
		return
	}

	t.Logf("EC-6 REPRODUCED: %v", err)

	// Verify rejection is specifically "wal_lag".
	pf, _ := r.EvaluatePromotion("ec6-vol")
	hasWALLag := false
	for _, rej := range pf.Rejections {
		t.Logf("  rejected %s: %s", rej.Server, rej.Reason)
		if rej.Reason == "wal_lag" {
			hasWALLag = true
		}
	}
	if !hasWALLag {
		t.Fatal("expected wal_lag rejection")
	}

	t.Log("EC-6: volume stuck — replica has 70% of data but promotion blocked by stale primary LSN")
}

func TestEC6_ForcePromote_Bypasses(t *testing.T) {
	r := NewBlockVolumeRegistry()
	r.MarkBlockCapable("primary:9333")
	r.MarkBlockCapable("replica:9333")

	r.Register(&BlockVolumeEntry{
		Name:           "ec6-force",
		VolumeServer:   "primary:9333",
		Path:           "/data/ec6-force.blk",
		SizeBytes:      1 << 30,
		Epoch:          1,
		Role:           blockvol.RoleToWire(blockvol.RolePrimary),
		Status:         StatusActive,
		LeaseTTL:       30 * time.Second,
		LastLeaseGrant: time.Now(),
		WALHeadLSN:     500,
		Replicas: []ReplicaInfo{
			{
				Server:        "replica:9333",
				Path:          "/data/ec6-force.blk",
				Role:          blockvol.RoleToWire(blockvol.RoleReplica),
				WALHeadLSN:    350,
				HealthScore:   1.0,
				LastHeartbeat: time.Now(),
			},
		},
	})

	r.UnmarkBlockCapable("primary:9333")

	// Force promote bypasses wal_lag gate.
	newEpoch, _, _, _, err := r.ManualPromote("ec6-force", "", true)
	if err != nil {
		t.Fatalf("force promote should succeed: %v", err)
	}

	entry, _ := r.Lookup("ec6-force")
	if entry.VolumeServer != "replica:9333" {
		t.Fatalf("primary=%s, want replica:9333", entry.VolumeServer)
	}

	t.Logf("EC-6 workaround: force promote succeeded — epoch=%d, new primary=%s", newEpoch, entry.VolumeServer)
}

func TestEC6_WithinTolerance_Succeeds(t *testing.T) {
	r := NewBlockVolumeRegistry()
	r.MarkBlockCapable("primary:9333")
	r.MarkBlockCapable("replica:9333")

	r.Register(&BlockVolumeEntry{
		Name:           "ec6-ok",
		VolumeServer:   "primary:9333",
		Path:           "/data/ec6-ok.blk",
		SizeBytes:      1 << 30,
		Epoch:          1,
		Role:           blockvol.RoleToWire(blockvol.RolePrimary),
		Status:         StatusActive,
		LeaseTTL:       30 * time.Second,
		LastLeaseGrant: time.Now(),
		WALHeadLSN:     500,
		Replicas: []ReplicaInfo{
			{
				Server:        "replica:9333",
				Path:          "/data/ec6-ok.blk",
				Role:          blockvol.RoleToWire(blockvol.RoleReplica),
				WALHeadLSN:    450, // 50 behind — within tolerance
				HealthScore:   1.0,
				LastHeartbeat: time.Now(),
			},
		},
	})

	r.UnmarkBlockCapable("primary:9333")

	_, err := r.PromoteBestReplica("ec6-ok")
	if err != nil {
		t.Fatalf("within tolerance should succeed: %v", err)
	}

	t.Log("EC-6 within tolerance: lag=50 (< 100) → promotion succeeded")
}

// --- EC-5: Wrong primary after master restart ---

func TestEC5_FirstHeartbeatWins(t *testing.T) {
	r := NewBlockVolumeRegistry()
	r.MarkBlockCapable("replica:9333")
	r.MarkBlockCapable("primary:9333")

	// Replica heartbeats FIRST after master restart.
	r.UpdateFullHeartbeat("replica:9333", []*master_pb.BlockVolumeInfoMessage{
		{
			Path:       "/data/ec5-vol.blk",
			VolumeSize: 1 << 30,
			Epoch:      0, // no assignment yet
			Role:       0, // unknown
			WalHeadLsn: 100,
		},
	}, "")

	entry1, ok := r.Lookup("ec5-vol")
	if !ok {
		t.Fatal("volume should be auto-registered from first heartbeat")
	}
	t.Logf("after replica heartbeat: VolumeServer=%s epoch=%d", entry1.VolumeServer, entry1.Epoch)

	firstPrimary := entry1.VolumeServer

	// Real primary heartbeats SECOND with higher epoch and WAL.
	r.UpdateFullHeartbeat("primary:9333", []*master_pb.BlockVolumeInfoMessage{
		{
			Path:       "/data/ec5-vol.blk",
			VolumeSize: 1 << 30,
			Epoch:      1,
			Role:       blockvol.RoleToWire(blockvol.RolePrimary),
			WalHeadLsn: 200,
		},
	}, "")

	entry2, _ := r.Lookup("ec5-vol")
	t.Logf("after primary heartbeat: VolumeServer=%s epoch=%d WALHeadLSN=%d",
		entry2.VolumeServer, entry2.Epoch, entry2.WALHeadLSN)

	if entry2.VolumeServer == "replica:9333" && firstPrimary == "replica:9333" {
		t.Log("EC-5 REPRODUCED: first-heartbeat-wins race — replica is still primary")
		t.Log("Real primary (epoch=1, WAL=200) was ignored or reconciled incorrectly")
	} else if entry2.VolumeServer == "primary:9333" {
		t.Log("EC-5 mitigated: reconcileOnRestart corrected the primary via epoch/WAL comparison")
	} else {
		t.Logf("EC-5 unknown: VolumeServer=%s", entry2.VolumeServer)
	}
}