seaweedfs/weed/server/qa_promote_rejoin_live_test.go

// Reproduces the live cp11b3-manual-promote failure path:
// create RF=2 → kill primary ��� promote replica → restart killed VS → verify
// that the promoted primary eventually gets an assignment with replica addresses.
//
// This test simulates the exact heartbeat/assignment sequence observed in glog
// to find where replicas are dropped.
package weed_server

import (
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	blockvol "github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)

func TestPromote_LivePath_RestartedVSRejoinsAndPrimaryGetsReplica(t *testing.T) {
	ms := testMasterServerForFailover(t)

	pathVS1 := "/data/vs1/promote-test.blk"
	pathVS2 := "/data/vs2/promote-test.blk"

	ms.blockRegistry.MarkBlockCapable("vs1:18192")
	ms.blockRegistry.MarkBlockCapable("vs2:18193")

	// Step 1: Create RF=2 volume. VS1=primary, VS2=replica.
	ms.blockRegistry.Register(&BlockVolumeEntry{
		Name:           "promote-test",
		VolumeServer:   "vs1:18192",
		Path:           pathVS1,
		ISCSIAddr:      "vs1:3279",
		SizeBytes:      50 << 20,
		Epoch:          1,
		Role:           blockvol.RoleToWire(blockvol.RolePrimary),
		Status:         StatusActive,
		LeaseTTL:       30 * time.Second,
		LastLeaseGrant: time.Now(),
		ReplicaFactor:  2,
		ReplicaServer:  "vs2:18193",
		ReplicaPath:    pathVS2,
		Replicas: []ReplicaInfo{
			{
				Server:        "vs2:18193",
				Path:          pathVS2,
				ISCSIAddr:     "vs2:3280",
				HealthScore:   1.0,
				Role:          blockvol.RoleToWire(blockvol.RoleReplica),
				LastHeartbeat: time.Now(),
				DataAddr:      "vs2:14260",
				CtrlAddr:      "vs2:14261",
			},
		},
	})

	t.Log("Step 1: created RF=2, vs1=primary, vs2=replica")

	// Step 2: Kill VS1 (simulate disconnect).
	ms.blockRegistry.UnmarkBlockCapable("vs1:18192")

	// Step 3: Expire lease and promote VS2.
	ms.blockRegistry.UpdateEntry("promote-test", func(e *BlockVolumeEntry) {
		e.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
	})
	ms.failoverBlockVolumes("vs1:18192")

	entry := lookupEntryT(t, ms.blockRegistry, "promote-test")
	if entry.VolumeServer != "vs2:18193" {
		t.Fatalf("promote failed: primary=%s", entry.VolumeServer)
	}
	t.Logf("Step 3: promoted vs2, epoch=%d, replicas=%d", entry.Epoch, len(entry.Replicas))

	// Drain the promote assignment.
	promoteAssignments := ms.blockAssignmentQueue.Peek("vs2:18193")
	for _, a := range promoteAssignments {
		ms.blockAssignmentQueue.Confirm("vs2:18193", a.Path, a.Epoch)
	}
	t.Logf("Step 3: drained %d promote assignments for vs2", len(promoteAssignments))

	// Step 4: VS1 restarts. First heartbeat: epoch=1, role=primary (stale).
	ms.blockRegistry.MarkBlockCapable("vs1:18192")
	result := ms.blockRegistry.UpdateFullHeartbeat("vs1:18192", []*master_pb.BlockVolumeInfoMessage{
		{
			Path:       pathVS1,
			VolumeSize: 50 << 20,
			Epoch:      1,
			Role:       blockvol.RoleToWire(blockvol.RolePrimary),
			// First heartbeat: no replica addresses yet.
		},
	}, "")

	entry = lookupEntryT(t, ms.blockRegistry, "promote-test")
	t.Logf("Step 4a: after vs1 first heartbeat: primary=%s replicas=%d", entry.VolumeServer, len(entry.Replicas))
	for i, ri := range entry.Replicas {
		t.Logf("  replica[%d]: server=%s data=%s ctrl=%s", i, ri.Server, ri.DataAddr, ri.CtrlAddr)
	}

	// Process any primary refresh from first heartbeat.
	for _, refresh := range result.PrimaryRefreshNeeded {
		ms.enqueuePrimaryRefresh(refresh)
		t.Logf("Step 4a: primary refresh triggered for %s", refresh.Name)
	}

	// Check if vs2 got a refresh assignment with replicas.
	vs2Assignments := ms.blockAssignmentQueue.Peek("vs2:18193")
	t.Logf("Step 4a: vs2 pending assignments: %d", len(vs2Assignments))
	for _, a := range vs2Assignments {
		t.Logf("  assignment: path=%s epoch=%d role=%d replicaDataAddr=%s replicaAddrs=%d",
			a.Path, a.Epoch, a.Role, a.ReplicaDataAddr, len(a.ReplicaAddrs))
	}

	// Step 5: VS1 second heartbeat: now with replica data/ctrl addresses.
	result2 := ms.blockRegistry.UpdateFullHeartbeat("vs1:18192", []*master_pb.BlockVolumeInfoMessage{
		{
			Path:            pathVS1,
			VolumeSize:      50 << 20,
			Epoch:           1,
			Role:            blockvol.RoleToWire(blockvol.RolePrimary),
			ReplicaDataAddr: "vs1:14262",
			ReplicaCtrlAddr: "vs1:14263",
		},
	}, "")

	entry = lookupEntryT(t, ms.blockRegistry, "promote-test")
	t.Logf("Step 5: after vs1 second heartbeat (with addrs): primary=%s replicas=%d", entry.VolumeServer, len(entry.Replicas))
	for i, ri := range entry.Replicas {
		t.Logf("  replica[%d]: server=%s data=%s ctrl=%s", i, ri.Server, ri.DataAddr, ri.CtrlAddr)
	}

	// Process any primary refresh from second heartbeat.
	for _, refresh := range result2.PrimaryRefreshNeeded {
		ms.enqueuePrimaryRefresh(refresh)
		t.Logf("Step 5: primary refresh triggered for %s", refresh.Name)
	}

	// Step 6: Final check — vs2 should now have a pending assignment with replica addresses.
	vs2Final := ms.blockAssignmentQueue.Peek("vs2:18193")
	t.Logf("Step 6: vs2 final pending assignments: %d", len(vs2Final))

	hasReplicaAddrs := false
	for _, a := range vs2Final {
		t.Logf("  assignment: path=%s epoch=%d role=%d replicaDataAddr=%s replicaAddrs=%d",
			a.Path, a.Epoch, a.Role, a.ReplicaDataAddr, len(a.ReplicaAddrs))
		if a.ReplicaDataAddr != "" || len(a.ReplicaAddrs) > 0 {
			hasReplicaAddrs = true
		}
	}

	if !hasReplicaAddrs {
		t.Fatalf("BUG: after promote + vs1 rejoin with addresses, vs2 STILL has no assignment with replica addresses.\n"+
			"This reproduces the live cp11b3-manual-promote failure: promoted primary never gets shipper configuration.")
	}
	t.Log("SUCCESS: promoted primary has pending assignment with replica addresses after rejoin")
}

func TestPromote_LivePath_FirstHeartbeatThenRecoverUsesDeterministicReplicaAddrs(t *testing.T) {
	ms := testMasterServerForFailover(t)

	pathVS1 := "/data/vs1/promote-test.blk"
	pathVS2 := "/data/vs2/promote-test.blk"

	ms.blockRegistry.MarkBlockCapable("vs1:18192")
	ms.blockRegistry.MarkBlockCapable("vs2:18193")

	ms.blockRegistry.Register(&BlockVolumeEntry{
		Name:           "promote-test",
		VolumeServer:   "vs1:18192",
		Path:           pathVS1,
		ISCSIAddr:      "192.168.1.184:3279",
		SizeBytes:      50 << 20,
		Epoch:          1,
		Role:           blockvol.RoleToWire(blockvol.RolePrimary),
		Status:         StatusActive,
		LeaseTTL:       30 * time.Second,
		LastLeaseGrant: time.Now().Add(-1 * time.Minute),
		ReplicaFactor:  2,
		ReplicaServer:  "vs2:18193",
		ReplicaPath:    pathVS2,
		Replicas: []ReplicaInfo{{
			Server:        "vs2:18193",
			Path:          pathVS2,
			ISCSIAddr:     "192.168.1.184:3280",
			HealthScore:   1.0,
			Role:          blockvol.RoleToWire(blockvol.RoleReplica),
			LastHeartbeat: time.Now(),
			DataAddr:      "192.168.1.184:4571",
			CtrlAddr:      "192.168.1.184:4572",
		}},
	})

	ms.blockRegistry.UnmarkBlockCapable("vs1:18192")
	ms.failoverBlockVolumes("vs1:18192")

	for _, a := range ms.blockAssignmentQueue.Peek("vs2:18193") {
		ms.blockAssignmentQueue.Confirm("vs2:18193", a.Path, a.Epoch)
	}

	// The restarted old primary first reports only a stale heartbeat with no
	// receiver addrs, matching the live manual-promote evidence.
	ms.blockRegistry.MarkBlockCapable("vs1:18192")
	ms.blockRegistry.UpdateFullHeartbeat("vs1:18192", []*master_pb.BlockVolumeInfoMessage{{
		Path:       pathVS1,
		VolumeSize: 50 << 20,
		Epoch:      1,
		Role:       blockvol.RoleToWire(blockvol.RolePrimary),
	}}, "")

	// Live path: after processing the heartbeat, the master runs reconnect
	// recovery immediately. The pending rebuild must already carry deterministic
	// replica addrs so recoverBlockVolumes can take the catch-up-first path
	// without waiting for a second heartbeat.
	ms.recoverBlockVolumes("vs1:18192")

	vs1Assignments := ms.blockAssignmentQueue.Peek("vs1:18192")
	foundReplicaAssign := false
	for _, a := range vs1Assignments {
		if a.Path != pathVS1 {
			continue
		}
		if blockvol.RoleFromWire(a.Role) == blockvol.RoleReplica && a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" {
			foundReplicaAssign = true
		}
		if blockvol.RoleFromWire(a.Role) == blockvol.RoleRebuilding {
			t.Fatalf("expected catch-up replica assignment, got rebuild assignment: %+v", a)
		}
	}
	if !foundReplicaAssign {
		t.Fatalf("restarted VS1 did not receive replica assignment with deterministic receiver addrs")
	}

	vs2Assignments := ms.blockAssignmentQueue.Peek("vs2:18193")
	foundPrimaryRefresh := false
	for _, a := range vs2Assignments {
		if a.Path != pathVS2 {
			continue
		}
		if blockvol.RoleFromWire(a.Role) == blockvol.RolePrimary && (a.ReplicaDataAddr != "" || len(a.ReplicaAddrs) > 0) {
			foundPrimaryRefresh = true
		}
	}
	if !foundPrimaryRefresh {
		t.Fatalf("promoted primary did not receive refreshed assignment with replica membership after reconnect recovery")
	}
}