seaweedfs/weed/storage/blockvol/csi/nvme_node_test.go

package csi

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// --- Transport selection tests ---

func TestNodeStageVolume_NVMe(t *testing.T) {
	mi := newMockISCSIUtil()
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.getDeviceResult = "/dev/nvme0n1"
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iqnPrefix: "iqn.2024.com.seaweedfs",
		nqnPrefix: "nqn.2024-01.com.seaweedfs:vol.",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "nvme-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:nvme-vol",
			"nvmeAddr":  "10.0.0.5:4420",
			"nqn":       "nqn.2024.com.seaweedfs:nvme-vol",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// Verify NVMe connect was called.
	foundConnect := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "connect:nqn.2024.com.seaweedfs:nvme-vol:10.0.0.5:4420") {
			foundConnect = true
		}
	}
	if !foundConnect {
		t.Fatalf("expected NVMe connect call, got: %v", mn.calls)
	}

	// Verify iSCSI was NOT called.
	if len(mi.calls) > 0 {
		t.Fatalf("expected no iSCSI calls, got: %v", mi.calls)
	}

	// Verify transport is NVMe.
	ns.stagedMu.Lock()
	info := ns.staged["nvme-vol"]
	ns.stagedMu.Unlock()
	if info == nil {
		t.Fatal("expected nvme-vol in staged map")
	}
	if info.transport != transportNVMe {
		t.Fatalf("expected transport=nvme, got %q", info.transport)
	}

	// Verify .transport file was written.
	data, err := os.ReadFile(filepath.Join(stagingPath, transportFile))
	if err != nil {
		t.Fatalf("read transport file: %v", err)
	}
	if string(data) != "nvme" {
		t.Fatalf("transport file: got %q, want nvme", string(data))
	}
}

func TestNodeStageVolume_NVMe_FallbackISCSI_ModuleNotLoaded(t *testing.T) {
	mi := newMockISCSIUtil()
	mi.getDeviceResult = "/dev/sdb"
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = false // nvme_tcp NOT loaded
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "fallback-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:fallback-vol",
			"nvmeAddr":  "10.0.0.5:4420",
			"nqn":       "nqn.2024.com.seaweedfs:fallback-vol",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// Should have used iSCSI.
	if len(mi.calls) == 0 {
		t.Fatal("expected iSCSI calls")
	}
	if len(mn.calls) > 0 {
		t.Fatalf("expected no NVMe calls, got: %v", mn.calls)
	}

	// Verify transport is iSCSI.
	ns.stagedMu.Lock()
	info := ns.staged["fallback-vol"]
	ns.stagedMu.Unlock()
	if info.transport != transportISCSI {
		t.Fatalf("expected transport=iscsi, got %q", info.transport)
	}
}

func TestNodeStageVolume_NVMe_ConnectFails_NoFallback(t *testing.T) {
	mi := newMockISCSIUtil()
	mi.getDeviceResult = "/dev/sdb"
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.connectErr = errors.New("connect refused")
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "fail-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:fail-vol",
			"nvmeAddr":  "10.0.0.5:4420",
			"nqn":       "nqn.2024.com.seaweedfs:fail-vol",
		},
	})
	if err == nil {
		t.Fatal("expected error when NVMe connect fails")
	}

	// Verify error and no fallback to iSCSI.
	st, _ := status.FromError(err)
	if st.Code() != codes.Internal {
		t.Fatalf("expected Internal error, got %v", st.Code())
	}
	if !strings.Contains(err.Error(), "nvme connect") {
		t.Fatalf("expected nvme connect in error, got: %v", err)
	}

	// iSCSI should NOT have been called (no fallback).
	if len(mi.calls) > 0 {
		t.Fatalf("expected no iSCSI fallback calls, got: %v", mi.calls)
	}
}

func TestNodeStageVolume_ISCSIOnly(t *testing.T) {
	mi := newMockISCSIUtil()
	mi.getDeviceResult = "/dev/sdb"
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	// No NVMe fields in context → iSCSI only.
	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "iscsi-only",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:iscsi-only",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// iSCSI should be used.
	if len(mi.calls) == 0 {
		t.Fatal("expected iSCSI calls")
	}
	if len(mn.calls) > 0 {
		t.Fatalf("expected no NVMe calls, got: %v", mn.calls)
	}

	ns.stagedMu.Lock()
	info := ns.staged["iscsi-only"]
	ns.stagedMu.Unlock()
	if info.transport != transportISCSI {
		t.Fatalf("expected transport=iscsi, got %q", info.transport)
	}
}

func TestNodeStageVolume_TransportStickiness(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.getDeviceResult = "/dev/nvme0n1"
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	// First stage via NVMe.
	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "sticky-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:sticky-vol",
			"nvmeAddr":  "10.0.0.5:4420",
			"nqn":       "nqn.2024.com.seaweedfs:sticky-vol",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// Verify staged as NVMe.
	ns.stagedMu.Lock()
	info := ns.staged["sticky-vol"]
	ns.stagedMu.Unlock()
	if info.transport != transportNVMe {
		t.Fatalf("expected transport=nvme, got %q", info.transport)
	}

	// Re-stage attempt (already mounted) → idempotent, transport stays.
	mm.isMountedTargets[stagingPath] = true
	_, err = ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "sticky-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:sticky-vol",
			// No NVMe fields this time.
		},
	})
	if err != nil {
		t.Fatalf("re-stage: %v", err)
	}

	// Transport should still be NVMe from first stage.
	ns.stagedMu.Lock()
	info = ns.staged["sticky-vol"]
	ns.stagedMu.Unlock()
	if info.transport != transportNVMe {
		t.Fatalf("expected transport=nvme after re-stage, got %q", info.transport)
	}
}

// --- Idempotency tests ---

func TestNodeStageVolume_NVMe_AlreadyConnected(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.connected["nqn.2024.com.seaweedfs:precon-vol"] = true
	mn.getDeviceResult = "/dev/nvme0n1"
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "precon-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"nvmeAddr": "10.0.0.5:4420",
			"nqn":      "nqn.2024.com.seaweedfs:precon-vol",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// Connect should NOT have been called (already connected).
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "connect:") {
			t.Fatalf("expected no connect call (already connected), got: %v", mn.calls)
		}
	}
}

func TestNodeStageVolume_NVMe_AlreadyMounted(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()
	mm.isMountedTargets[stagingPath] = true // already mounted

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "mounted-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"nvmeAddr": "10.0.0.5:4420",
			"nqn":      "nqn.2024.com.seaweedfs:mounted-vol",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// No NVMe or iSCSI calls should have been made.
	if len(mn.calls) > 0 {
		t.Fatalf("expected no NVMe calls, got: %v", mn.calls)
	}
	if len(mi.calls) > 0 {
		t.Fatalf("expected no iSCSI calls, got: %v", mi.calls)
	}
}

func TestNodeStageVolume_NVMe_ConcurrentSameVolume(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.getDeviceResult = "/dev/nvme0n1"
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()
	ctx := context.Background()

	var wg sync.WaitGroup
	errs := make([]error, 2)
	for i := 0; i < 2; i++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			_, errs[idx] = ns.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
				VolumeId:          "concurrent-vol",
				StagingTargetPath: stagingPath,
				VolumeCapability:  testVolCap(),
				PublishContext: map[string]string{
					"nvmeAddr": "10.0.0.5:4420",
					"nqn":      "nqn.2024.com.seaweedfs:concurrent-vol",
				},
			})
		}(i)
	}
	wg.Wait()

	// At least one should succeed.
	success := false
	for _, err := range errs {
		if err == nil {
			success = true
		}
	}
	if !success {
		t.Fatalf("expected at least one concurrent stage to succeed: %v, %v", errs[0], errs[1])
	}
}

// --- Unstage tests ---

func TestNodeUnstageVolume_NVMe(t *testing.T) {
	mn := newMockNVMeUtil()
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged: map[string]*stagedVolumeInfo{
			"unstage-vol": {
				nqn:       "nqn.2024.com.seaweedfs:unstage-vol",
				nvmeAddr:  "10.0.0.5:4420",
				transport: transportNVMe,
			},
		},
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{
		VolumeId:          "unstage-vol",
		StagingTargetPath: stagingPath,
	})
	if err != nil {
		t.Fatalf("NodeUnstageVolume: %v", err)
	}

	// Verify NVMe disconnect was called.
	foundDisconnect := false
	for _, c := range mn.calls {
		if c == "disconnect:nqn.2024.com.seaweedfs:unstage-vol" {
			foundDisconnect = true
		}
	}
	if !foundDisconnect {
		t.Fatalf("expected NVMe disconnect call, got: %v", mn.calls)
	}

	// Verify iSCSI logout was NOT called.
	for _, c := range mi.calls {
		if strings.HasPrefix(c, "logout:") {
			t.Fatalf("expected no iSCSI logout for NVMe volume, got: %v", mi.calls)
		}
	}
}

func TestNodeUnstageVolume_NVMe_RestartFallback_TransportFile(t *testing.T) {
	mn := newMockNVMeUtil()
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		nqnPrefix: "nqn.2024-01.com.seaweedfs:vol.",
		iqnPrefix: "iqn.2024.com.seaweedfs",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo), // empty (restart)
	}

	// Write a .transport file indicating NVMe.
	stagingPath := t.TempDir()
	os.WriteFile(filepath.Join(stagingPath, transportFile), []byte("nvme"), 0600)

	_, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{
		VolumeId:          "restart-nvme-vol",
		StagingTargetPath: stagingPath,
	})
	if err != nil {
		t.Fatalf("NodeUnstageVolume: %v", err)
	}

	// Verify NVMe disconnect was called (derived NQN from nqnPrefix).
	foundDisconnect := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "disconnect:") {
			foundDisconnect = true
		}
	}
	if !foundDisconnect {
		t.Fatalf("expected NVMe disconnect from transport file, got: %v", mn.calls)
	}
}

func TestNodeUnstageVolume_NVMe_RestartFallback_Probe(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.connected["nqn.2024-01.com.seaweedfs:vol.probe-vol"] = true
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		nqnPrefix: "nqn.2024-01.com.seaweedfs:vol.",
		iqnPrefix: "iqn.2024.com.seaweedfs",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo), // empty (restart)
	}

	stagingPath := t.TempDir()
	// No .transport file — will probe NVMe connection.

	_, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{
		VolumeId:          "probe-vol",
		StagingTargetPath: stagingPath,
	})
	if err != nil {
		t.Fatalf("NodeUnstageVolume: %v", err)
	}

	// Verify NVMe disconnect was called (probe found connection).
	foundDisconnect := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "disconnect:") {
			foundDisconnect = true
		}
	}
	if !foundDisconnect {
		t.Fatalf("expected NVMe disconnect from probe, got: %v", mn.calls)
	}
}

// --- Expand tests ---

func TestNodeExpandVolume_NVMe(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.getDeviceResult = "/dev/nvme0n1"
	mn.getControllerResult = "/dev/nvme0"
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged: map[string]*stagedVolumeInfo{
			"expand-vol": {
				nqn:         "nqn.2024.com.seaweedfs:expand-vol",
				transport:   transportNVMe,
				fsType:      "ext4",
				stagingPath: "/staging/expand-vol",
			},
		},
	}

	_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
		VolumeId:      "expand-vol",
		CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
	})
	if err != nil {
		t.Fatalf("NodeExpandVolume: %v", err)
	}

	// Verify NVMe rescan was called.
	foundRescan := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "rescan:") {
			foundRescan = true
		}
	}
	if !foundRescan {
		t.Fatalf("expected NVMe rescan call, got: %v", mn.calls)
	}

	// Verify resize was called.
	foundResize := false
	for _, c := range mm.calls {
		if strings.HasPrefix(c, "resizefs:") {
			foundResize = true
		}
	}
	if !foundResize {
		t.Fatalf("expected resizefs call, got: %v", mm.calls)
	}

	// Verify iSCSI rescan was NOT called.
	for _, c := range mi.calls {
		if strings.HasPrefix(c, "rescan:") {
			t.Fatalf("expected no iSCSI rescan for NVMe volume, got: %v", mi.calls)
		}
	}
}

// --- Failure cleanup tests ---

func TestNodeStageVolume_NVMe_GetDeviceFails_Cleanup(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.getDeviceErr = errors.New("device not found")
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "cleanup-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"nvmeAddr": "10.0.0.5:4420",
			"nqn":      "nqn.2024.com.seaweedfs:cleanup-vol",
		},
	})
	if err == nil {
		t.Fatal("expected error when GetDeviceByNQN fails")
	}

	// Verify Disconnect was called in cleanup.
	foundDisconnect := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "disconnect:") {
			foundDisconnect = true
		}
	}
	if !foundDisconnect {
		t.Fatalf("expected Disconnect in cleanup after GetDevice failure, got: %v", mn.calls)
	}
}

func TestNodeStageVolume_NVMe_FormatFails_Cleanup(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.nvmeTCPAvailable = true
	mn.getDeviceResult = "/dev/nvme0n1"
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()
	mm.formatAndMountErr = errors.New("mkfs failed")

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "format-fail-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"nvmeAddr": "10.0.0.5:4420",
			"nqn":      "nqn.2024.com.seaweedfs:format-fail-vol",
		},
	})
	if err == nil {
		t.Fatal("expected error when format fails")
	}

	// Verify Disconnect was called in cleanup.
	foundDisconnect := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "disconnect:") {
			foundDisconnect = true
		}
	}
	if !foundDisconnect {
		t.Fatalf("expected Disconnect in cleanup after format failure, got: %v", mn.calls)
	}
}

// --- Controller/VolumeInfo tests ---

func TestControllerPublish_NVMeFields(t *testing.T) {
	mgr := newTestManager(t)

	backend := NewLocalVolumeBackend(mgr)
	cs := &controllerServer{backend: backend}

	if err := mgr.CreateVolume("nvme-pub-vol", 4*1024*1024); err != nil {
		t.Fatalf("create volume: %v", err)
	}

	resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
		VolumeId: "nvme-pub-vol",
		NodeId:   "node-1",
	})
	if err != nil {
		t.Fatalf("ControllerPublishVolume: %v", err)
	}

	// iSCSI fields should always be present.
	if resp.PublishContext["iscsiAddr"] == "" {
		t.Fatal("expected iscsiAddr in PublishContext")
	}
	if resp.PublishContext["iqn"] == "" {
		t.Fatal("expected iqn in PublishContext")
	}

	// NVMe fields are empty when VolumeManager has no NVMe config.
	if resp.PublishContext["nvmeAddr"] != "" {
		t.Fatalf("expected no nvmeAddr (NVMe not configured), got %q", resp.PublishContext["nvmeAddr"])
	}
}

func TestControllerPublish_NoNVMe(t *testing.T) {
	mgr := newTestManager(t)

	backend := NewLocalVolumeBackend(mgr)
	cs := &controllerServer{backend: backend}

	if err := mgr.CreateVolume("iscsi-only-vol", 4*1024*1024); err != nil {
		t.Fatalf("create volume: %v", err)
	}

	resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
		VolumeId: "iscsi-only-vol",
		NodeId:   "node-1",
	})
	if err != nil {
		t.Fatalf("ControllerPublishVolume: %v", err)
	}

	// Only iSCSI fields, no NVMe keys.
	if _, ok := resp.PublishContext["nvmeAddr"]; ok {
		t.Fatal("expected no nvmeAddr key in PublishContext when NVMe not configured")
	}
	if _, ok := resp.PublishContext["nqn"]; ok {
		t.Fatal("expected no nqn key in PublishContext when NVMe not configured")
	}
}

func TestCreateVolume_NVMeContext(t *testing.T) {
	dir := t.TempDir()
	logger := log.New(os.Stderr, "[test-vm] ", log.LstdFlags)
	mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger,
		VolumeManagerOpts{NvmeAddr: "10.0.0.5:4420", NQNPrefix: "nqn.2024-01.com.seaweedfs:vol."})
	if err := mgr.Start(context.Background()); err != nil {
		t.Fatalf("start: %v", err)
	}
	t.Cleanup(func() { mgr.Stop() })

	backend := NewLocalVolumeBackend(mgr)
	cs := &controllerServer{backend: backend}

	resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
		Name:               "nvme-ctx-vol",
		VolumeCapabilities: testVolCaps(),
		CapacityRange:      &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
	})
	if err != nil {
		t.Fatalf("CreateVolume: %v", err)
	}

	vc := resp.Volume.VolumeContext
	if vc["nvmeAddr"] != "10.0.0.5:4420" {
		t.Fatalf("expected nvmeAddr=10.0.0.5:4420, got %q", vc["nvmeAddr"])
	}
	if !strings.HasPrefix(vc["nqn"], "nqn.2024-01.com.seaweedfs:vol.") {
		t.Fatalf("expected nqn prefix, got %q", vc["nqn"])
	}
}

// --- Parity and edge-case tests (review findings) ---

// TestNQNParity_CSIAndBlockService verifies that CSI VolumeManager and BlockService
// produce identical NQNs for the same volume name with the same prefix.
func TestNQNParity_CSIAndBlockService(t *testing.T) {
	prefix := "nqn.2024-01.com.seaweedfs:vol."
	names := []string{
		"simple-vol",
		"pvc-abc123-def456",
		"UPPERCASE-Volume",
		"special.chars-here_ok",
		"a-very-long-volume-name-that-might-need-truncation-or-hashing-to-stay-within-limits-xxxxxxxxx",
	}

	for _, name := range names {
		// CSI path: VolumeManager.volumeNQN → blockvol.BuildNQN(prefix, name)
		dir := t.TempDir()
		logger := log.New(os.Stderr, "[test] ", log.LstdFlags)
		mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger,
			VolumeManagerOpts{NQNPrefix: prefix})
		if err := mgr.Start(context.Background()); err != nil {
			t.Fatalf("start: %v", err)
		}
		csiNQN := mgr.VolumeNQN(name)
		mgr.Stop()

		// BlockService path: blockvol.BuildNQN(prefix, name)
		// This is what volume_server_block.go:registerVolume and NQN() use.
		blockNQN := SanitizeIQN(name) // prefix + SanitizeIQN
		blockNQN = prefix + blockNQN

		if csiNQN != blockNQN {
			t.Errorf("NQN mismatch for %q: CSI=%q BlockService=%q", name, csiNQN, blockNQN)
		}
	}
}

// TestIsConnected_ErrorClassification verifies that IsConnected distinguishes
// "NQN not found" from command/parse failures.
func TestIsConnected_ErrorClassification(t *testing.T) {
	// Test with errNQNNotFound — should return false, nil.
	r := &realNVMeUtil{}

	// We can't run real nvme commands on Windows, so we test the sentinel logic directly.
	// Verify errNQNNotFound is a distinct sentinel.
	if !errors.Is(errNQNNotFound, errNQNNotFound) {
		t.Fatal("errNQNNotFound sentinel should be identifiable via errors.Is")
	}

	// Verify it's different from a wrapped exec error.
	execErr := fmt.Errorf("nvme list-subsys: command not found: %w", errors.New("exit code 127"))
	if errors.Is(execErr, errNQNNotFound) {
		t.Fatal("exec error should NOT match errNQNNotFound")
	}

	// Verify the mock distinguishes correctly.
	mn := newMockNVMeUtil()

	// Not connected (NQN absent) → (false, nil).
	connected, err := mn.IsConnected(context.Background(), "nqn.absent")
	if err != nil {
		t.Fatalf("expected no error for absent NQN, got: %v", err)
	}
	if connected {
		t.Fatal("expected not connected for absent NQN")
	}

	// Connected → (true, nil).
	mn.connected["nqn.present"] = true
	connected, err = mn.IsConnected(context.Background(), "nqn.present")
	if err != nil {
		t.Fatalf("expected no error: %v", err)
	}
	if !connected {
		t.Fatal("expected connected")
	}

	_ = r // suppress unused warning, real path tested via integration on Linux
}

// TestFindSubsys_Fixture verifies findSubsys path selection with a multi-path JSON fixture.
func TestFindSubsys_Fixture(t *testing.T) {
	// Test the JSON parsing and path selection logic directly.
	fixture := `{
		"Subsystems": [
			{
				"NQN": "nqn.2024-01.com.seaweedfs:vol.test-vol",
				"Paths": [
					{"Name": "nvme1", "Transport": "rdma", "State": "live"},
					{"Name": "nvme2", "Transport": "tcp", "State": "connecting"},
					{"Name": "nvme3", "Transport": "tcp", "State": "live"},
					{"Name": "", "Transport": "tcp", "State": "live"}
				]
			},
			{
				"NQN": "nqn.other:subsystem",
				"Paths": [
					{"Name": "nvme0", "Transport": "tcp", "State": "live"}
				]
			}
		]
	}`

	var parsed nvmeListSubsysOutput
	if err := json.Unmarshal([]byte(fixture), &parsed); err != nil {
		t.Fatalf("parse fixture: %v", err)
	}

	// Simulate findSubsys logic for target NQN.
	nqn := "nqn.2024-01.com.seaweedfs:vol.test-vol"
	var foundCtrl, foundDev string
	for _, ss := range parsed.Subsystems {
		if ss.NQN != nqn {
			continue
		}
		var fallbackCtrl string
		for _, p := range ss.Paths {
			if p.Name == "" {
				continue
			}
			ctrl := "/dev/" + p.Name
			dev := ctrl + "n1"
			if strings.EqualFold(p.Transport, "tcp") && strings.EqualFold(p.State, "live") {
				foundCtrl = ctrl
				foundDev = dev
				break
			}
			if fallbackCtrl == "" {
				fallbackCtrl = ctrl
			}
		}
		if foundCtrl == "" && fallbackCtrl != "" {
			foundCtrl = fallbackCtrl
			foundDev = fallbackCtrl + "n1"
		}
	}

	// Should prefer nvme3 (tcp+live), not nvme1 (rdma) or nvme2 (connecting).
	if foundCtrl != "/dev/nvme3" {
		t.Fatalf("expected /dev/nvme3 (tcp+live), got %q", foundCtrl)
	}
	if foundDev != "/dev/nvme3n1" {
		t.Fatalf("expected /dev/nvme3n1, got %q", foundDev)
	}

	// Test NQN not found.
	found := false
	for _, ss := range parsed.Subsystems {
		if ss.NQN == "nqn.nonexistent" {
			found = true
		}
	}
	if found {
		t.Fatal("nonexistent NQN should not be found")
	}
}

// TestMasterBackend_NVMeFieldsAbsent verifies that MasterVolumeClient
// returns empty NVMe fields (deferred scope per CP10-2 design).
func TestMasterBackend_NVMeFieldsAbsent(t *testing.T) {
	// We can't call real master gRPC, so verify the struct directly.
	// The point: VolumeInfo from MasterVolumeClient has empty NvmeAddr/NQN.
	info := &VolumeInfo{
		VolumeID:  "master-vol",
		ISCSIAddr: "10.0.0.5:3260",
		IQN:       "iqn.2024.com.seaweedfs:master-vol",
		// NvmeAddr and NQN intentionally not set (master proto lacks these fields).
	}

	if info.NvmeAddr != "" {
		t.Fatalf("expected empty NvmeAddr from master backend, got %q", info.NvmeAddr)
	}
	if info.NQN != "" {
		t.Fatalf("expected empty NQN from master backend, got %q", info.NQN)
	}

	// Verify controller doesn't add NVMe keys when fields are empty.
	cs := &controllerServer{backend: &staticBackend{info: info}}
	resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
		VolumeId: "master-vol",
		NodeId:   "node-1",
	})
	if err != nil {
		t.Fatalf("ControllerPublishVolume: %v", err)
	}
	if _, ok := resp.PublishContext["nvmeAddr"]; ok {
		t.Fatal("expected no nvmeAddr in PublishContext for master-backed volume")
	}
	if _, ok := resp.PublishContext["nqn"]; ok {
		t.Fatal("expected no nqn in PublishContext for master-backed volume")
	}
}

// --- Issue 4: nvmeUtil == nil with NVMe metadata ---

func TestNodeStageVolume_NVMeUtilNil_FallsToISCSI(t *testing.T) {
	mi := newMockISCSIUtil()
	mi.getDeviceResult = "/dev/sdb"
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  nil, // NVMe util not available
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged:    make(map[string]*stagedVolumeInfo),
	}

	stagingPath := t.TempDir()

	// Both NVMe and iSCSI metadata present, but nvmeUtil is nil → must use iSCSI.
	_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
		VolumeId:          "nil-util-vol",
		StagingTargetPath: stagingPath,
		VolumeCapability:  testVolCap(),
		PublishContext: map[string]string{
			"iscsiAddr": "10.0.0.5:3260",
			"iqn":       "iqn.2024.com.seaweedfs:nil-util-vol",
			"nvmeAddr":  "10.0.0.5:4420",
			"nqn":       "nqn.2024.com.seaweedfs:nil-util-vol",
		},
	})
	if err != nil {
		t.Fatalf("NodeStageVolume: %v", err)
	}

	// iSCSI should be used.
	if len(mi.calls) == 0 {
		t.Fatal("expected iSCSI calls when nvmeUtil is nil")
	}

	ns.stagedMu.Lock()
	info := ns.staged["nil-util-vol"]
	ns.stagedMu.Unlock()
	if info.transport != transportISCSI {
		t.Fatalf("expected transport=iscsi, got %q", info.transport)
	}
}

// --- Missing test: Expand with NVMe Rescan failure ---

func TestNodeExpandVolume_NVMe_RescanFails(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.rescanErr = errors.New("ns-rescan failed")
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged: map[string]*stagedVolumeInfo{
			"rescan-fail-vol": {
				nqn:         "nqn.2024.com.seaweedfs:rescan-fail-vol",
				transport:   transportNVMe,
				fsType:      "ext4",
				stagingPath: "/staging/rescan-fail-vol",
			},
		},
	}

	_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
		VolumeId:      "rescan-fail-vol",
		CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
	})
	if err == nil {
		t.Fatal("expected error when NVMe rescan fails")
	}
	st, _ := status.FromError(err)
	if st.Code() != codes.Internal {
		t.Fatalf("expected Internal, got %v", st.Code())
	}
	if !strings.Contains(err.Error(), "nvme rescan") {
		t.Fatalf("expected 'nvme rescan' in error, got: %v", err)
	}
}

// --- Missing test: Unstage NVMe disconnect failure ---

func TestNodeUnstageVolume_NVMe_DisconnectFails(t *testing.T) {
	mn := newMockNVMeUtil()
	mn.disconnectErr = errors.New("disconnect timeout")
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged: map[string]*stagedVolumeInfo{
			"disc-fail-vol": {
				nqn:       "nqn.2024.com.seaweedfs:disc-fail-vol",
				nvmeAddr:  "10.0.0.5:4420",
				transport: transportNVMe,
			},
		},
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{
		VolumeId:          "disc-fail-vol",
		StagingTargetPath: stagingPath,
	})
	// Should return error (disconnect failure is surfaced).
	if err == nil {
		t.Fatal("expected error when NVMe disconnect fails")
	}
	if !strings.Contains(err.Error(), "disconnect timeout") {
		t.Fatalf("expected disconnect error, got: %v", err)
	}

	// Volume should remain in staged map for retry.
	ns.stagedMu.Lock()
	_, stillStaged := ns.staged["disc-fail-vol"]
	ns.stagedMu.Unlock()
	if !stillStaged {
		t.Fatal("volume should remain in staged map after disconnect failure")
	}
}

// --- Missing test: Disconnect idempotency (already disconnected) ---

func TestNodeUnstageVolume_NVMe_AlreadyDisconnected(t *testing.T) {
	mn := newMockNVMeUtil()
	// No disconnect error — idempotent success even though not connected.
	mi := newMockISCSIUtil()
	mm := newMockMountUtil()

	ns := &nodeServer{
		nodeID:    "test-node-1",
		iscsiUtil: mi,
		nvmeUtil:  mn,
		mountUtil: mm,
		logger:    log.New(os.Stderr, "[test-nvme] ", log.LstdFlags),
		staged: map[string]*stagedVolumeInfo{
			"already-disc-vol": {
				nqn:       "nqn.2024.com.seaweedfs:already-disc-vol",
				nvmeAddr:  "10.0.0.5:4420",
				transport: transportNVMe,
			},
		},
	}

	stagingPath := t.TempDir()

	_, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{
		VolumeId:          "already-disc-vol",
		StagingTargetPath: stagingPath,
	})
	if err != nil {
		t.Fatalf("NodeUnstageVolume: %v", err)
	}

	// Disconnect should be called (idempotent) and succeed.
	foundDisconnect := false
	for _, c := range mn.calls {
		if strings.HasPrefix(c, "disconnect:") {
			foundDisconnect = true
		}
	}
	if !foundDisconnect {
		t.Fatalf("expected disconnect call, got: %v", mn.calls)
	}
}

// --- Missing test: writeTransportFile error handling ---

func TestWriteTransportFile_Error(t *testing.T) {
	// Write to a non-existent directory → should return error.
	err := writeTransportFile("/nonexistent/path/that/does/not/exist", transportNVMe)
	if err == nil {
		t.Fatal("expected error writing to non-existent directory")
	}
	if !strings.Contains(err.Error(), "write transport file") {
		t.Fatalf("expected wrapped error, got: %v", err)
	}
}

// staticBackend is a minimal VolumeBackend for testing controller behavior.
type staticBackend struct {
	info *VolumeInfo
}

func (b *staticBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) {
	return b.info, nil
}
func (b *staticBackend) DeleteVolume(_ context.Context, name string) error { return nil }
func (b *staticBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) {
	return b.info, nil
}
func (b *staticBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) {
	return nil, nil
}
func (b *staticBackend) DeleteSnapshot(_ context.Context, volumeID string, snapID uint32) error {
	return nil
}
func (b *staticBackend) ListSnapshots(_ context.Context, volumeID string) ([]*SnapshotInfo, error) {
	return nil, nil
}
func (b *staticBackend) ExpandVolume(_ context.Context, volumeID string, newSizeBytes uint64) (uint64, error) {
	return 0, nil
}