seaweedfs/weed/server/qa_block_publication_test.go

package weed_server

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)

// ============================================================
// Phase 11 P3: Front-end publication rebinding
//
// Proofs:
//   1. Create/Lookup coherence: iSCSI + NVMe fields match
//   2. Failover: publication truth switches to new primary
//   3. Heartbeat reconstruction: fields survive restart
//   4. No-NVMe fallback: explicit and coherent
//
// V1 reuse roles:
//   master_grpc_server_block.go: reuse as bounded adapter
//   master_block_registry.go: reuse as bounded truth carrier
//   volume_server_block.go: reuse as publication source only
// ============================================================

func newPublicationMaster(t *testing.T, nvmeEnabled bool) *MasterServer {
	t.Helper()
	dir := t.TempDir()

	ms := &MasterServer{
		blockRegistry:        NewBlockVolumeRegistry(),
		blockAssignmentQueue: NewBlockAssignmentQueue(),
		blockFailover:        newBlockFailoverState(),
	}
	ms.blockRegistry.MarkBlockCapable("vs1:9333")
	ms.blockRegistry.MarkBlockCapable("vs2:9333")

	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
		sanitized := strings.ReplaceAll(server, ":", "_")
		serverDir := filepath.Join(dir, sanitized)
		os.MkdirAll(serverDir, 0755)
		volPath := filepath.Join(serverDir, fmt.Sprintf("%s.blk", name))
		vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{
			VolumeSize: 1 * 1024 * 1024,
			BlockSize:  4096,
			WALSize:    256 * 1024,
		})
		if err != nil {
			return nil, err
		}
		vol.Close()

		host := server
		if idx := strings.LastIndex(server, ":"); idx >= 0 {
			host = server[:idx]
		}

		result := &blockAllocResult{
			Path:              volPath,
			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
			ISCSIAddr:         host + ":3260",
			ReplicaDataAddr:   server + ":14260",
			ReplicaCtrlAddr:   server + ":14261",
			RebuildListenAddr: server + ":15000",
		}
		if nvmeEnabled {
			result.NvmeAddr = host + ":4420"
			result.NQN = fmt.Sprintf("nqn.2024-01.com.seaweedfs:vol.%s", name)
		}
		return result, nil
	}
	ms.blockVSDelete = func(ctx context.Context, server string, name string) error { return nil }

	return ms
}

// --- 1. Create/Lookup coherence ---

func TestP11P3_CreateLookup_PublicationCoherence(t *testing.T) {
	ms := newPublicationMaster(t, true)
	ctx := context.Background()

	createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
		Name: "pub-vol-1", SizeBytes: 1 << 30,
	})
	if err != nil {
		t.Fatalf("Create: %v", err)
	}

	// Create response has iSCSI + NVMe fields.
	if createResp.IscsiAddr == "" || createResp.Iqn == "" {
		t.Fatal("create missing iSCSI fields")
	}
	if createResp.NvmeAddr == "" || createResp.Nqn == "" {
		t.Fatal("create missing NVMe fields")
	}

	// Lookup must return the SAME publication fields.
	lookupResp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-1"})
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}

	if lookupResp.IscsiAddr != createResp.IscsiAddr {
		t.Fatalf("iSCSI addr mismatch: create=%q lookup=%q", createResp.IscsiAddr, lookupResp.IscsiAddr)
	}
	if lookupResp.Iqn != createResp.Iqn {
		t.Fatalf("IQN mismatch: create=%q lookup=%q", createResp.Iqn, lookupResp.Iqn)
	}
	if lookupResp.NvmeAddr != createResp.NvmeAddr {
		t.Fatalf("NVMe addr mismatch: create=%q lookup=%q", createResp.NvmeAddr, lookupResp.NvmeAddr)
	}
	if lookupResp.Nqn != createResp.Nqn {
		t.Fatalf("NQN mismatch: create=%q lookup=%q", createResp.Nqn, lookupResp.Nqn)
	}

	// Verify: registry truth matches.
	entry, _ := ms.blockRegistry.Lookup("pub-vol-1")
	if entry.ISCSIAddr != createResp.IscsiAddr {
		t.Fatalf("registry iSCSI=%q != create=%q", entry.ISCSIAddr, createResp.IscsiAddr)
	}
	if entry.NvmeAddr != createResp.NvmeAddr {
		t.Fatalf("registry NVMe=%q != create=%q", entry.NvmeAddr, createResp.NvmeAddr)
	}

	t.Logf("P11P3 coherence: create=lookup=registry — iSCSI=%s NVMe=%s IQN=%s NQN=%s",
		createResp.IscsiAddr, createResp.NvmeAddr, createResp.Iqn, createResp.Nqn)
}

// --- 2. Failover: publication truth switches ---

func TestP11P3_Failover_PublicationSwitches(t *testing.T) {
	ms := newPublicationMaster(t, true)
	ctx := context.Background()

	createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
		Name: "pub-vol-2", SizeBytes: 1 << 30,
	})
	if err != nil {
		t.Fatalf("Create: %v", err)
	}

	oldPrimary := createResp.VolumeServer
	oldISCSI := createResp.IscsiAddr
	oldNVMe := createResp.NvmeAddr

	t.Logf("before failover: primary=%s iSCSI=%s NVMe=%s", oldPrimary, oldISCSI, oldNVMe)

	// Expire lease and failover.
	ms.blockRegistry.UpdateEntry("pub-vol-2", func(e *BlockVolumeEntry) {
		e.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
	})
	ms.failoverBlockVolumes(oldPrimary)

	// Lookup after failover: publication fields should reflect NEW primary.
	lookupResp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-2"})
	if err != nil {
		t.Fatalf("Lookup after failover: %v", err)
	}

	entry, _ := ms.blockRegistry.Lookup("pub-vol-2")
	if entry.VolumeServer == oldPrimary {
		t.Fatalf("primary should have changed, still %s", oldPrimary)
	}

	// New primary's publication fields should differ from old primary's.
	if lookupResp.IscsiAddr == oldISCSI {
		t.Fatalf("iSCSI addr should change after failover: still %q", lookupResp.IscsiAddr)
	}
	if lookupResp.NvmeAddr == oldNVMe {
		t.Fatalf("NVMe addr should change after failover: still %q", lookupResp.NvmeAddr)
	}

	// Registry and lookup must still agree.
	if lookupResp.IscsiAddr != entry.ISCSIAddr {
		t.Fatalf("post-failover: lookup iSCSI=%q != registry=%q", lookupResp.IscsiAddr, entry.ISCSIAddr)
	}
	if lookupResp.NvmeAddr != entry.NvmeAddr {
		t.Fatalf("post-failover: lookup NVMe=%q != registry=%q", lookupResp.NvmeAddr, entry.NvmeAddr)
	}

	t.Logf("P11P3 failover: old=%s→new=%s iSCSI=%s NVMe=%s (switched)",
		oldPrimary, entry.VolumeServer, lookupResp.IscsiAddr, lookupResp.NvmeAddr)
}

// --- 3. Heartbeat reconstruction ---

func TestP11P3_HeartbeatReconstruction(t *testing.T) {
	ms := newPublicationMaster(t, true)
	ctx := context.Background()

	createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
		Name: "pub-vol-3", SizeBytes: 1 << 30,
	})
	if err != nil {
		t.Fatalf("Create: %v", err)
	}

	entry, _ := ms.blockRegistry.Lookup("pub-vol-3")
	originalNVMe := entry.NvmeAddr
	originalNQN := entry.NQN
	primaryServer := entry.VolumeServer

	// Clear NVMe fields to simulate master restart state loss.
	ms.blockRegistry.UpdateEntry("pub-vol-3", func(e *BlockVolumeEntry) {
		e.NvmeAddr = ""
		e.NQN = ""
	})

	cleared, _ := ms.blockRegistry.Lookup("pub-vol-3")
	if cleared.NvmeAddr != "" {
		t.Fatal("NvmeAddr should be cleared")
	}

	// Reconstruct via REAL heartbeat path: UpdateFullHeartbeat with proto info.
	// This is the same code path as master_grpc_server.go:280.
	heartbeatInfo := &master_pb.BlockVolumeInfoMessage{
		Path:           entry.Path,
		VolumeSize:     entry.SizeBytes,
		BlockSize:      4096,
		Epoch:          entry.Epoch,
		Role:           entry.Role,
		NvmeAddr:       originalNVMe,
		Nqn:            originalNQN,
	}

	// Derive server-level NVMe addr (same as what the VS heartbeat carries).
	host := primaryServer
	if idx := strings.LastIndex(host, ":"); idx >= 0 {
		host = host[:idx]
	}
	serverNvmeAddr := host + ":4420"

	ms.blockRegistry.UpdateFullHeartbeat(
		primaryServer,
		[]*master_pb.BlockVolumeInfoMessage{heartbeatInfo},
		serverNvmeAddr,
	)

	// Verify: lookup reflects heartbeat-reconstructed NVMe truth.
	lookupResp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-3"})
	if err != nil {
		t.Fatalf("Lookup after reconstruction: %v", err)
	}
	if lookupResp.NvmeAddr != originalNVMe {
		t.Fatalf("reconstructed NvmeAddr=%q, want %q", lookupResp.NvmeAddr, originalNVMe)
	}
	if lookupResp.Nqn != originalNQN {
		t.Fatalf("reconstructed NQN=%q, want %q", lookupResp.Nqn, originalNQN)
	}

	_ = createResp // used for initial creation

	t.Logf("P11P3 reconstruction: cleared → UpdateFullHeartbeat → lookup restored NVMe=%s NQN=%s",
		lookupResp.NvmeAddr, lookupResp.Nqn)
}

// --- 4. No-NVMe fallback ---

func TestP11P3_NoNVMe_Fallback(t *testing.T) {
	ms := newPublicationMaster(t, false) // NVMe disabled
	ctx := context.Background()

	createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
		Name: "pub-vol-4", SizeBytes: 1 << 30,
	})
	if err != nil {
		t.Fatalf("Create: %v", err)
	}

	// iSCSI must be present.
	if createResp.IscsiAddr == "" || createResp.Iqn == "" {
		t.Fatal("iSCSI fields must be present even without NVMe")
	}

	// NVMe must be explicitly empty (not fabricated).
	if createResp.NvmeAddr != "" {
		t.Fatalf("NvmeAddr=%q should be empty when NVMe disabled", createResp.NvmeAddr)
	}
	if createResp.Nqn != "" {
		t.Fatalf("Nqn=%q should be empty when NVMe disabled", createResp.Nqn)
	}

	// Lookup must also show empty NVMe.
	lookupResp, _ := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-4"})
	if lookupResp.NvmeAddr != "" || lookupResp.Nqn != "" {
		t.Fatalf("lookup: NVMe should be empty, got addr=%q nqn=%q", lookupResp.NvmeAddr, lookupResp.Nqn)
	}

	// Registry must also show empty NVMe.
	entry, _ := ms.blockRegistry.Lookup("pub-vol-4")
	if entry.NvmeAddr != "" || entry.NQN != "" {
		t.Fatalf("registry: NVMe should be empty, got addr=%q nqn=%q", entry.NvmeAddr, entry.NQN)
	}

	t.Logf("P11P3 no-NVMe: iSCSI=%s IQN=%s, NVMe explicitly empty across create/lookup/registry",
		createResp.IscsiAddr, createResp.Iqn)
}