package csi import ( "context" "encoding/json" "errors" "fmt" "log" "os" "path/filepath" "strings" "sync" "testing" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // --- Transport selection tests --- func TestNodeStageVolume_NVMe(t *testing.T) { mi := newMockISCSIUtil() mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.getDeviceResult = "/dev/nvme0n1" mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", nqnPrefix: "nqn.2024-01.com.seaweedfs:vol.", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "nvme-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:nvme-vol", "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:nvme-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Verify NVMe connect was called. foundConnect := false for _, c := range mn.calls { if strings.HasPrefix(c, "connect:nqn.2024.com.seaweedfs:nvme-vol:10.0.0.5:4420") { foundConnect = true } } if !foundConnect { t.Fatalf("expected NVMe connect call, got: %v", mn.calls) } // Verify iSCSI was NOT called. if len(mi.calls) > 0 { t.Fatalf("expected no iSCSI calls, got: %v", mi.calls) } // Verify transport is NVMe. ns.stagedMu.Lock() info := ns.staged["nvme-vol"] ns.stagedMu.Unlock() if info == nil { t.Fatal("expected nvme-vol in staged map") } if info.transport != transportNVMe { t.Fatalf("expected transport=nvme, got %q", info.transport) } // Verify .transport file was written. data, err := os.ReadFile(filepath.Join(stagingPath, transportFile)) if err != nil { t.Fatalf("read transport file: %v", err) } if string(data) != "nvme" { t.Fatalf("transport file: got %q, want nvme", string(data)) } } func TestNodeStageVolume_NVMe_FallbackISCSI_ModuleNotLoaded(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mn := newMockNVMeUtil() mn.nvmeTCPAvailable = false // nvme_tcp NOT loaded mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "fallback-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:fallback-vol", "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:fallback-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Should have used iSCSI. if len(mi.calls) == 0 { t.Fatal("expected iSCSI calls") } if len(mn.calls) > 0 { t.Fatalf("expected no NVMe calls, got: %v", mn.calls) } // Verify transport is iSCSI. ns.stagedMu.Lock() info := ns.staged["fallback-vol"] ns.stagedMu.Unlock() if info.transport != transportISCSI { t.Fatalf("expected transport=iscsi, got %q", info.transport) } } func TestNodeStageVolume_NVMe_ConnectFails_NoFallback(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.connectErr = errors.New("connect refused") mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "fail-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:fail-vol", "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:fail-vol", }, }) if err == nil { t.Fatal("expected error when NVMe connect fails") } // Verify error and no fallback to iSCSI. st, _ := status.FromError(err) if st.Code() != codes.Internal { t.Fatalf("expected Internal error, got %v", st.Code()) } if !strings.Contains(err.Error(), "nvme connect") { t.Fatalf("expected nvme connect in error, got: %v", err) } // iSCSI should NOT have been called (no fallback). if len(mi.calls) > 0 { t.Fatalf("expected no iSCSI fallback calls, got: %v", mi.calls) } } func TestNodeStageVolume_ISCSIOnly(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() // No NVMe fields in context → iSCSI only. _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "iscsi-only", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:iscsi-only", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // iSCSI should be used. if len(mi.calls) == 0 { t.Fatal("expected iSCSI calls") } if len(mn.calls) > 0 { t.Fatalf("expected no NVMe calls, got: %v", mn.calls) } ns.stagedMu.Lock() info := ns.staged["iscsi-only"] ns.stagedMu.Unlock() if info.transport != transportISCSI { t.Fatalf("expected transport=iscsi, got %q", info.transport) } } func TestNodeStageVolume_TransportStickiness(t *testing.T) { mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.getDeviceResult = "/dev/nvme0n1" mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() // First stage via NVMe. _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "sticky-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:sticky-vol", "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:sticky-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Verify staged as NVMe. ns.stagedMu.Lock() info := ns.staged["sticky-vol"] ns.stagedMu.Unlock() if info.transport != transportNVMe { t.Fatalf("expected transport=nvme, got %q", info.transport) } // Re-stage attempt (already mounted) → idempotent, transport stays. mm.isMountedTargets[stagingPath] = true _, err = ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "sticky-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:sticky-vol", // No NVMe fields this time. }, }) if err != nil { t.Fatalf("re-stage: %v", err) } // Transport should still be NVMe from first stage. ns.stagedMu.Lock() info = ns.staged["sticky-vol"] ns.stagedMu.Unlock() if info.transport != transportNVMe { t.Fatalf("expected transport=nvme after re-stage, got %q", info.transport) } } // --- Idempotency tests --- func TestNodeStageVolume_NVMe_AlreadyConnected(t *testing.T) { mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.connected["nqn.2024.com.seaweedfs:precon-vol"] = true mn.getDeviceResult = "/dev/nvme0n1" mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "precon-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:precon-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Connect should NOT have been called (already connected). for _, c := range mn.calls { if strings.HasPrefix(c, "connect:") { t.Fatalf("expected no connect call (already connected), got: %v", mn.calls) } } } func TestNodeStageVolume_NVMe_AlreadyMounted(t *testing.T) { mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() mm.isMountedTargets[stagingPath] = true // already mounted _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "mounted-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:mounted-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // No NVMe or iSCSI calls should have been made. if len(mn.calls) > 0 { t.Fatalf("expected no NVMe calls, got: %v", mn.calls) } if len(mi.calls) > 0 { t.Fatalf("expected no iSCSI calls, got: %v", mi.calls) } } func TestNodeStageVolume_NVMe_ConcurrentSameVolume(t *testing.T) { mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.getDeviceResult = "/dev/nvme0n1" mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() ctx := context.Background() var wg sync.WaitGroup errs := make([]error, 2) for i := 0; i < 2; i++ { wg.Add(1) go func(idx int) { defer wg.Done() _, errs[idx] = ns.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ VolumeId: "concurrent-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:concurrent-vol", }, }) }(i) } wg.Wait() // At least one should succeed. success := false for _, err := range errs { if err == nil { success = true } } if !success { t.Fatalf("expected at least one concurrent stage to succeed: %v, %v", errs[0], errs[1]) } } // --- Unstage tests --- func TestNodeUnstageVolume_NVMe(t *testing.T) { mn := newMockNVMeUtil() mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "unstage-vol": { nqn: "nqn.2024.com.seaweedfs:unstage-vol", nvmeAddr: "10.0.0.5:4420", transport: transportNVMe, }, }, } stagingPath := t.TempDir() _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "unstage-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Verify NVMe disconnect was called. foundDisconnect := false for _, c := range mn.calls { if c == "disconnect:nqn.2024.com.seaweedfs:unstage-vol" { foundDisconnect = true } } if !foundDisconnect { t.Fatalf("expected NVMe disconnect call, got: %v", mn.calls) } // Verify iSCSI logout was NOT called. for _, c := range mi.calls { if strings.HasPrefix(c, "logout:") { t.Fatalf("expected no iSCSI logout for NVMe volume, got: %v", mi.calls) } } } func TestNodeUnstageVolume_NVMe_RestartFallback_TransportFile(t *testing.T) { mn := newMockNVMeUtil() mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", nqnPrefix: "nqn.2024-01.com.seaweedfs:vol.", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), // empty (restart) } // Write a .transport file indicating NVMe. stagingPath := t.TempDir() os.WriteFile(filepath.Join(stagingPath, transportFile), []byte("nvme"), 0600) _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "restart-nvme-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Verify NVMe disconnect was called (derived NQN from nqnPrefix). foundDisconnect := false for _, c := range mn.calls { if strings.HasPrefix(c, "disconnect:") { foundDisconnect = true } } if !foundDisconnect { t.Fatalf("expected NVMe disconnect from transport file, got: %v", mn.calls) } } func TestNodeUnstageVolume_NVMe_RestartFallback_Probe(t *testing.T) { mn := newMockNVMeUtil() mn.connected["nqn.2024-01.com.seaweedfs:vol.probe-vol"] = true mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", nqnPrefix: "nqn.2024-01.com.seaweedfs:vol.", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), // empty (restart) } stagingPath := t.TempDir() // No .transport file — will probe NVMe connection. _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "probe-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Verify NVMe disconnect was called (probe found connection). foundDisconnect := false for _, c := range mn.calls { if strings.HasPrefix(c, "disconnect:") { foundDisconnect = true } } if !foundDisconnect { t.Fatalf("expected NVMe disconnect from probe, got: %v", mn.calls) } } // --- Expand tests --- func TestNodeExpandVolume_NVMe(t *testing.T) { mn := newMockNVMeUtil() mn.getDeviceResult = "/dev/nvme0n1" mn.getControllerResult = "/dev/nvme0" mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "expand-vol": { nqn: "nqn.2024.com.seaweedfs:expand-vol", transport: transportNVMe, fsType: "ext4", stagingPath: "/staging/expand-vol", }, }, } _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ VolumeId: "expand-vol", CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, }) if err != nil { t.Fatalf("NodeExpandVolume: %v", err) } // Verify NVMe rescan was called. foundRescan := false for _, c := range mn.calls { if strings.HasPrefix(c, "rescan:") { foundRescan = true } } if !foundRescan { t.Fatalf("expected NVMe rescan call, got: %v", mn.calls) } // Verify resize was called. foundResize := false for _, c := range mm.calls { if strings.HasPrefix(c, "resizefs:") { foundResize = true } } if !foundResize { t.Fatalf("expected resizefs call, got: %v", mm.calls) } // Verify iSCSI rescan was NOT called. for _, c := range mi.calls { if strings.HasPrefix(c, "rescan:") { t.Fatalf("expected no iSCSI rescan for NVMe volume, got: %v", mi.calls) } } } // --- Failure cleanup tests --- func TestNodeStageVolume_NVMe_GetDeviceFails_Cleanup(t *testing.T) { mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.getDeviceErr = errors.New("device not found") mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "cleanup-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:cleanup-vol", }, }) if err == nil { t.Fatal("expected error when GetDeviceByNQN fails") } // Verify Disconnect was called in cleanup. foundDisconnect := false for _, c := range mn.calls { if strings.HasPrefix(c, "disconnect:") { foundDisconnect = true } } if !foundDisconnect { t.Fatalf("expected Disconnect in cleanup after GetDevice failure, got: %v", mn.calls) } } func TestNodeStageVolume_NVMe_FormatFails_Cleanup(t *testing.T) { mn := newMockNVMeUtil() mn.nvmeTCPAvailable = true mn.getDeviceResult = "/dev/nvme0n1" mi := newMockISCSIUtil() mm := newMockMountUtil() mm.formatAndMountErr = errors.New("mkfs failed") ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "format-fail-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:format-fail-vol", }, }) if err == nil { t.Fatal("expected error when format fails") } // Verify Disconnect was called in cleanup. foundDisconnect := false for _, c := range mn.calls { if strings.HasPrefix(c, "disconnect:") { foundDisconnect = true } } if !foundDisconnect { t.Fatalf("expected Disconnect in cleanup after format failure, got: %v", mn.calls) } } // --- Controller/VolumeInfo tests --- func TestControllerPublish_NVMeFields(t *testing.T) { mgr := newTestManager(t) backend := NewLocalVolumeBackend(mgr) cs := &controllerServer{backend: backend} if err := mgr.CreateVolume("nvme-pub-vol", 4*1024*1024); err != nil { t.Fatalf("create volume: %v", err) } resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{ VolumeId: "nvme-pub-vol", NodeId: "node-1", }) if err != nil { t.Fatalf("ControllerPublishVolume: %v", err) } // iSCSI fields should always be present. if resp.PublishContext["iscsiAddr"] == "" { t.Fatal("expected iscsiAddr in PublishContext") } if resp.PublishContext["iqn"] == "" { t.Fatal("expected iqn in PublishContext") } // NVMe fields are empty when VolumeManager has no NVMe config. if resp.PublishContext["nvmeAddr"] != "" { t.Fatalf("expected no nvmeAddr (NVMe not configured), got %q", resp.PublishContext["nvmeAddr"]) } } func TestControllerPublish_NoNVMe(t *testing.T) { mgr := newTestManager(t) backend := NewLocalVolumeBackend(mgr) cs := &controllerServer{backend: backend} if err := mgr.CreateVolume("iscsi-only-vol", 4*1024*1024); err != nil { t.Fatalf("create volume: %v", err) } resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{ VolumeId: "iscsi-only-vol", NodeId: "node-1", }) if err != nil { t.Fatalf("ControllerPublishVolume: %v", err) } // Only iSCSI fields, no NVMe keys. if _, ok := resp.PublishContext["nvmeAddr"]; ok { t.Fatal("expected no nvmeAddr key in PublishContext when NVMe not configured") } if _, ok := resp.PublishContext["nqn"]; ok { t.Fatal("expected no nqn key in PublishContext when NVMe not configured") } } func TestCreateVolume_NVMeContext(t *testing.T) { dir := t.TempDir() logger := log.New(os.Stderr, "[test-vm] ", log.LstdFlags) mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger, VolumeManagerOpts{NvmeAddr: "10.0.0.5:4420", NQNPrefix: "nqn.2024-01.com.seaweedfs:vol."}) if err := mgr.Start(context.Background()); err != nil { t.Fatalf("start: %v", err) } t.Cleanup(func() { mgr.Stop() }) backend := NewLocalVolumeBackend(mgr) cs := &controllerServer{backend: backend} resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ Name: "nvme-ctx-vol", VolumeCapabilities: testVolCaps(), CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, }) if err != nil { t.Fatalf("CreateVolume: %v", err) } vc := resp.Volume.VolumeContext if vc["nvmeAddr"] != "10.0.0.5:4420" { t.Fatalf("expected nvmeAddr=10.0.0.5:4420, got %q", vc["nvmeAddr"]) } if !strings.HasPrefix(vc["nqn"], "nqn.2024-01.com.seaweedfs:vol.") { t.Fatalf("expected nqn prefix, got %q", vc["nqn"]) } } // --- Parity and edge-case tests (review findings) --- // TestNQNParity_CSIAndBlockService verifies that CSI VolumeManager and BlockService // produce identical NQNs for the same volume name with the same prefix. func TestNQNParity_CSIAndBlockService(t *testing.T) { prefix := "nqn.2024-01.com.seaweedfs:vol." names := []string{ "simple-vol", "pvc-abc123-def456", "UPPERCASE-Volume", "special.chars-here_ok", "a-very-long-volume-name-that-might-need-truncation-or-hashing-to-stay-within-limits-xxxxxxxxx", } for _, name := range names { // CSI path: VolumeManager.volumeNQN → blockvol.BuildNQN(prefix, name) dir := t.TempDir() logger := log.New(os.Stderr, "[test] ", log.LstdFlags) mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger, VolumeManagerOpts{NQNPrefix: prefix}) if err := mgr.Start(context.Background()); err != nil { t.Fatalf("start: %v", err) } csiNQN := mgr.VolumeNQN(name) mgr.Stop() // BlockService path: blockvol.BuildNQN(prefix, name) // This is what volume_server_block.go:registerVolume and NQN() use. blockNQN := SanitizeIQN(name) // prefix + SanitizeIQN blockNQN = prefix + blockNQN if csiNQN != blockNQN { t.Errorf("NQN mismatch for %q: CSI=%q BlockService=%q", name, csiNQN, blockNQN) } } } // TestIsConnected_ErrorClassification verifies that IsConnected distinguishes // "NQN not found" from command/parse failures. func TestIsConnected_ErrorClassification(t *testing.T) { // Test with errNQNNotFound — should return false, nil. r := &realNVMeUtil{} // We can't run real nvme commands on Windows, so we test the sentinel logic directly. // Verify errNQNNotFound is a distinct sentinel. if !errors.Is(errNQNNotFound, errNQNNotFound) { t.Fatal("errNQNNotFound sentinel should be identifiable via errors.Is") } // Verify it's different from a wrapped exec error. execErr := fmt.Errorf("nvme list-subsys: command not found: %w", errors.New("exit code 127")) if errors.Is(execErr, errNQNNotFound) { t.Fatal("exec error should NOT match errNQNNotFound") } // Verify the mock distinguishes correctly. mn := newMockNVMeUtil() // Not connected (NQN absent) → (false, nil). connected, err := mn.IsConnected(context.Background(), "nqn.absent") if err != nil { t.Fatalf("expected no error for absent NQN, got: %v", err) } if connected { t.Fatal("expected not connected for absent NQN") } // Connected → (true, nil). mn.connected["nqn.present"] = true connected, err = mn.IsConnected(context.Background(), "nqn.present") if err != nil { t.Fatalf("expected no error: %v", err) } if !connected { t.Fatal("expected connected") } _ = r // suppress unused warning, real path tested via integration on Linux } // TestFindSubsys_Fixture verifies findSubsys path selection with a multi-path JSON fixture. func TestFindSubsys_Fixture(t *testing.T) { // Test the JSON parsing and path selection logic directly. fixture := `{ "Subsystems": [ { "NQN": "nqn.2024-01.com.seaweedfs:vol.test-vol", "Paths": [ {"Name": "nvme1", "Transport": "rdma", "State": "live"}, {"Name": "nvme2", "Transport": "tcp", "State": "connecting"}, {"Name": "nvme3", "Transport": "tcp", "State": "live"}, {"Name": "", "Transport": "tcp", "State": "live"} ] }, { "NQN": "nqn.other:subsystem", "Paths": [ {"Name": "nvme0", "Transport": "tcp", "State": "live"} ] } ] }` var parsed nvmeListSubsysOutput if err := json.Unmarshal([]byte(fixture), &parsed); err != nil { t.Fatalf("parse fixture: %v", err) } // Simulate findSubsys logic for target NQN. nqn := "nqn.2024-01.com.seaweedfs:vol.test-vol" var foundCtrl, foundDev string for _, ss := range parsed.Subsystems { if ss.NQN != nqn { continue } var fallbackCtrl string for _, p := range ss.Paths { if p.Name == "" { continue } ctrl := "/dev/" + p.Name dev := ctrl + "n1" if strings.EqualFold(p.Transport, "tcp") && strings.EqualFold(p.State, "live") { foundCtrl = ctrl foundDev = dev break } if fallbackCtrl == "" { fallbackCtrl = ctrl } } if foundCtrl == "" && fallbackCtrl != "" { foundCtrl = fallbackCtrl foundDev = fallbackCtrl + "n1" } } // Should prefer nvme3 (tcp+live), not nvme1 (rdma) or nvme2 (connecting). if foundCtrl != "/dev/nvme3" { t.Fatalf("expected /dev/nvme3 (tcp+live), got %q", foundCtrl) } if foundDev != "/dev/nvme3n1" { t.Fatalf("expected /dev/nvme3n1, got %q", foundDev) } // Test NQN not found. found := false for _, ss := range parsed.Subsystems { if ss.NQN == "nqn.nonexistent" { found = true } } if found { t.Fatal("nonexistent NQN should not be found") } } // TestMasterBackend_NVMeFieldsAbsent verifies that MasterVolumeClient // returns empty NVMe fields (deferred scope per CP10-2 design). func TestMasterBackend_NVMeFieldsAbsent(t *testing.T) { // We can't call real master gRPC, so verify the struct directly. // The point: VolumeInfo from MasterVolumeClient has empty NvmeAddr/NQN. info := &VolumeInfo{ VolumeID: "master-vol", ISCSIAddr: "10.0.0.5:3260", IQN: "iqn.2024.com.seaweedfs:master-vol", // NvmeAddr and NQN intentionally not set (master proto lacks these fields). } if info.NvmeAddr != "" { t.Fatalf("expected empty NvmeAddr from master backend, got %q", info.NvmeAddr) } if info.NQN != "" { t.Fatalf("expected empty NQN from master backend, got %q", info.NQN) } // Verify controller doesn't add NVMe keys when fields are empty. cs := &controllerServer{backend: &staticBackend{info: info}} resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{ VolumeId: "master-vol", NodeId: "node-1", }) if err != nil { t.Fatalf("ControllerPublishVolume: %v", err) } if _, ok := resp.PublishContext["nvmeAddr"]; ok { t.Fatal("expected no nvmeAddr in PublishContext for master-backed volume") } if _, ok := resp.PublishContext["nqn"]; ok { t.Fatal("expected no nqn in PublishContext for master-backed volume") } } // --- Issue 4: nvmeUtil == nil with NVMe metadata --- func TestNodeStageVolume_NVMeUtilNil_FallsToISCSI(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: nil, // NVMe util not available mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() // Both NVMe and iSCSI metadata present, but nvmeUtil is nil → must use iSCSI. _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "nil-util-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:nil-util-vol", "nvmeAddr": "10.0.0.5:4420", "nqn": "nqn.2024.com.seaweedfs:nil-util-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // iSCSI should be used. if len(mi.calls) == 0 { t.Fatal("expected iSCSI calls when nvmeUtil is nil") } ns.stagedMu.Lock() info := ns.staged["nil-util-vol"] ns.stagedMu.Unlock() if info.transport != transportISCSI { t.Fatalf("expected transport=iscsi, got %q", info.transport) } } // --- Missing test: Expand with NVMe Rescan failure --- func TestNodeExpandVolume_NVMe_RescanFails(t *testing.T) { mn := newMockNVMeUtil() mn.rescanErr = errors.New("ns-rescan failed") mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "rescan-fail-vol": { nqn: "nqn.2024.com.seaweedfs:rescan-fail-vol", transport: transportNVMe, fsType: "ext4", stagingPath: "/staging/rescan-fail-vol", }, }, } _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ VolumeId: "rescan-fail-vol", CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, }) if err == nil { t.Fatal("expected error when NVMe rescan fails") } st, _ := status.FromError(err) if st.Code() != codes.Internal { t.Fatalf("expected Internal, got %v", st.Code()) } if !strings.Contains(err.Error(), "nvme rescan") { t.Fatalf("expected 'nvme rescan' in error, got: %v", err) } } // --- Missing test: Unstage NVMe disconnect failure --- func TestNodeUnstageVolume_NVMe_DisconnectFails(t *testing.T) { mn := newMockNVMeUtil() mn.disconnectErr = errors.New("disconnect timeout") mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "disc-fail-vol": { nqn: "nqn.2024.com.seaweedfs:disc-fail-vol", nvmeAddr: "10.0.0.5:4420", transport: transportNVMe, }, }, } stagingPath := t.TempDir() _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "disc-fail-vol", StagingTargetPath: stagingPath, }) // Should return error (disconnect failure is surfaced). if err == nil { t.Fatal("expected error when NVMe disconnect fails") } if !strings.Contains(err.Error(), "disconnect timeout") { t.Fatalf("expected disconnect error, got: %v", err) } // Volume should remain in staged map for retry. ns.stagedMu.Lock() _, stillStaged := ns.staged["disc-fail-vol"] ns.stagedMu.Unlock() if !stillStaged { t.Fatal("volume should remain in staged map after disconnect failure") } } // --- Missing test: Disconnect idempotency (already disconnected) --- func TestNodeUnstageVolume_NVMe_AlreadyDisconnected(t *testing.T) { mn := newMockNVMeUtil() // No disconnect error — idempotent success even though not connected. mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ nodeID: "test-node-1", iscsiUtil: mi, nvmeUtil: mn, mountUtil: mm, logger: log.New(os.Stderr, "[test-nvme] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "already-disc-vol": { nqn: "nqn.2024.com.seaweedfs:already-disc-vol", nvmeAddr: "10.0.0.5:4420", transport: transportNVMe, }, }, } stagingPath := t.TempDir() _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "already-disc-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Disconnect should be called (idempotent) and succeed. foundDisconnect := false for _, c := range mn.calls { if strings.HasPrefix(c, "disconnect:") { foundDisconnect = true } } if !foundDisconnect { t.Fatalf("expected disconnect call, got: %v", mn.calls) } } // --- Missing test: writeTransportFile error handling --- func TestWriteTransportFile_Error(t *testing.T) { // Write to a non-existent directory → should return error. err := writeTransportFile("/nonexistent/path/that/does/not/exist", transportNVMe) if err == nil { t.Fatal("expected error writing to non-existent directory") } if !strings.Contains(err.Error(), "write transport file") { t.Fatalf("expected wrapped error, got: %v", err) } } // staticBackend is a minimal VolumeBackend for testing controller behavior. type staticBackend struct { info *VolumeInfo } func (b *staticBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { return b.info, nil } func (b *staticBackend) DeleteVolume(_ context.Context, name string) error { return nil } func (b *staticBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) { return b.info, nil } func (b *staticBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { return nil, nil } func (b *staticBackend) DeleteSnapshot(_ context.Context, volumeID string, snapID uint32) error { return nil } func (b *staticBackend) ListSnapshots(_ context.Context, volumeID string) ([]*SnapshotInfo, error) { return nil, nil } func (b *staticBackend) ExpandVolume(_ context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { return 0, nil }