package csi import ( "context" "errors" "log" "os" "strings" "testing" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) func newTestNodeServer(t *testing.T) (*nodeServer, *mockISCSIUtil, *mockMountUtil) { t.Helper() mgr := newTestManager(t) // Pre-create a volume for stage/unstage tests. if err := mgr.CreateVolume("test-vol", 4*1024*1024); err != nil { t.Fatalf("create test-vol: %v", err) } // Close it so OpenVolume in NodeStageVolume can reopen it. if err := mgr.CloseVolume("test-vol"); err != nil { t.Fatalf("close test-vol: %v", err) } mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sda" mm := newMockMountUtil() ns := &nodeServer{ mgr: mgr, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } return ns, mi, mm } func TestNode_StageUnstage(t *testing.T) { ns, mi, mm := newTestNodeServer(t) stagingPath := t.TempDir() // Stage _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: &csi.VolumeCapability{ AccessType: &csi.VolumeCapability_Mount{ Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, }, }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Verify calls. if len(mi.calls) < 3 { t.Fatalf("expected at least 3 iscsi calls, got %d: %v", len(mi.calls), mi.calls) } if mi.calls[0] != "discovery:"+ns.mgr.ListenAddr() { t.Fatalf("expected discovery call, got %q", mi.calls[0]) } if len(mm.calls) < 1 { t.Fatalf("expected at least 1 mount call, got %d: %v", len(mm.calls), mm.calls) } // Verify staged map entry. ns.stagedMu.Lock() info, ok := ns.staged["test-vol"] ns.stagedMu.Unlock() if !ok { t.Fatal("expected test-vol in staged map") } if !info.isLocal { t.Fatal("expected isLocal=true for local volume") } // Unstage _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Verify staged map cleared. ns.stagedMu.Lock() _, ok = ns.staged["test-vol"] ns.stagedMu.Unlock() if ok { t.Fatal("expected test-vol removed from staged map") } } func TestNode_PublishUnpublish(t *testing.T) { ns, _, _ := newTestNodeServer(t) stagingPath := t.TempDir() targetPath := t.TempDir() _, err := ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, TargetPath: targetPath, VolumeCapability: &csi.VolumeCapability{ AccessType: &csi.VolumeCapability_Mount{ Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, }, }, }) if err != nil { t.Fatalf("NodePublishVolume: %v", err) } _, err = ns.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ VolumeId: "test-vol", TargetPath: targetPath, }) if err != nil { t.Fatalf("NodeUnpublishVolume: %v", err) } } func TestNode_StageIdempotent(t *testing.T) { ns, _, mm := newTestNodeServer(t) stagingPath := t.TempDir() // Mark as already mounted -> stage should be idempotent. mm.isMountedTargets[stagingPath] = true _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), }) if err != nil { t.Fatalf("idempotent NodeStageVolume: %v", err) } // No iscsi or mount calls should have been made. if len(mm.calls) != 0 { t.Fatalf("expected 0 mount calls, got %d: %v", len(mm.calls), mm.calls) } } func TestNode_StageLoginFailure(t *testing.T) { ns, mi, _ := newTestNodeServer(t) mi.loginErr = errors.New("connection refused") stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), }) if err == nil { t.Fatal("expected error from login failure") } st, ok := status.FromError(err) if !ok || st.Code() != codes.Internal { t.Fatalf("expected Internal error, got: %v", err) } } func TestNode_StageMkfsFailure(t *testing.T) { ns, _, mm := newTestNodeServer(t) mm.formatAndMountErr = errors.New("mkfs failed") stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), }) if err == nil { t.Fatal("expected error from mkfs failure") } st, ok := status.FromError(err) if !ok || st.Code() != codes.Internal { t.Fatalf("expected Internal error, got: %v", err) } } // TestNode_StageLoginFailureCleanup verifies that the volume is closed/disconnected // when login fails during NodeStageVolume (Finding #3: resource leak). func TestNode_StageLoginFailureCleanup(t *testing.T) { ns, mi, _ := newTestNodeServer(t) mi.loginErr = errors.New("connection refused") stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), }) if err == nil { t.Fatal("expected error from login failure") } // Volume should have been cleaned up (closed) after error. if ns.mgr.VolumeExists("test-vol") { t.Fatal("expected volume to be closed after login failure (resource leak)") } } // TestNode_PublishMissingStagingPath verifies that NodePublishVolume rejects // empty StagingTargetPath (Finding #2). func TestNode_PublishMissingStagingPath(t *testing.T) { ns, _, _ := newTestNodeServer(t) _, err := ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: "", TargetPath: t.TempDir(), }) if err == nil { t.Fatal("expected error for empty StagingTargetPath") } st, ok := status.FromError(err) if !ok || st.Code() != codes.InvalidArgument { t.Fatalf("expected InvalidArgument error, got: %v", err) } } // --- Remote target tests --- // TestNode_StageRemoteTarget verifies NodeStageVolume reads iSCSI info from volume_context // instead of using local VolumeManager. func TestNode_StageRemoteTarget(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, // no local manager nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "remote-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), VolumeContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:remote-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Verify discovery was called with remote portal. if len(mi.calls) < 1 || mi.calls[0] != "discovery:10.0.0.5:3260" { t.Fatalf("expected discovery with remote portal, got: %v", mi.calls) } // Verify staged map has remote info. ns.stagedMu.Lock() info, ok := ns.staged["remote-vol"] ns.stagedMu.Unlock() if !ok { t.Fatal("expected remote-vol in staged map") } if info.isLocal { t.Fatal("expected isLocal=false for remote volume") } if info.iqn != "iqn.2024.com.seaweedfs:remote-vol" { t.Fatalf("unexpected IQN: %s", info.iqn) } } // TestNode_UnstageRemoteTarget verifies unstage uses the staged map IQN. func TestNode_UnstageRemoteTarget(t *testing.T) { mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "remote-vol": { iqn: "iqn.2024.com.seaweedfs:remote-vol", iscsiAddr: "10.0.0.5:3260", isLocal: false, }, }, } stagingPath := t.TempDir() _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "remote-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Verify logout was called with correct IQN. foundLogout := false for _, c := range mi.calls { if c == "logout:iqn.2024.com.seaweedfs:remote-vol" { foundLogout = true } } if !foundLogout { t.Fatalf("expected logout call with remote IQN, got: %v", mi.calls) } } // TestNode_StagePrefersPublishContext verifies that publish_context takes priority // over volume_context (reflects current primary after failover). func TestNode_StagePrefersPublishContext(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() // publish_context has fresh address (after failover), volume_context has stale. _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "failover-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), PublishContext: map[string]string{ "iscsiAddr": "10.0.0.99:3260", "iqn": "iqn.2024.com.seaweedfs:failover-vol-new", }, VolumeContext: map[string]string{ "iscsiAddr": "10.0.0.1:3260", "iqn": "iqn.2024.com.seaweedfs:failover-vol-old", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Should have used publish_context (new primary address). if len(mi.calls) < 1 || mi.calls[0] != "discovery:10.0.0.99:3260" { t.Fatalf("expected discovery with publish_context portal, got: %v", mi.calls) } ns.stagedMu.Lock() info := ns.staged["failover-vol"] ns.stagedMu.Unlock() if info == nil { t.Fatal("expected failover-vol in staged map") } if info.iqn != "iqn.2024.com.seaweedfs:failover-vol-new" { t.Fatalf("expected IQN from publish_context, got %q", info.iqn) } if info.iscsiAddr != "10.0.0.99:3260" { t.Fatalf("expected iscsiAddr from publish_context, got %q", info.iscsiAddr) } } // TestNode_StageFallbackToVolumeContext verifies that volume_context is used // when publish_context is not set (backward compatibility). func TestNode_StageFallbackToVolumeContext(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), } stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "compat-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), VolumeContext: map[string]string{ "iscsiAddr": "10.0.0.5:3260", "iqn": "iqn.2024.com.seaweedfs:compat-vol", }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Should have used volume_context. if len(mi.calls) < 1 || mi.calls[0] != "discovery:10.0.0.5:3260" { t.Fatalf("expected discovery with volume_context portal, got: %v", mi.calls) } } // TestNode_UnstageAfterRestart verifies IQN derivation when staged map is empty. func TestNode_UnstageAfterRestart(t *testing.T) { mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: make(map[string]*stagedVolumeInfo), // empty (simulates restart) } stagingPath := t.TempDir() _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "restart-vol", StagingTargetPath: stagingPath, }) if err != nil { t.Fatalf("NodeUnstageVolume: %v", err) } // Verify logout was called with derived IQN. foundLogout := false for _, c := range mi.calls { if c == "logout:iqn.2024.com.seaweedfs:restart-vol" { foundLogout = true } } if !foundLogout { t.Fatalf("expected logout with derived IQN, got: %v", mi.calls) } } // TestNode_UnstageRetryKeepsStagedEntry verifies that if unmount fails, // the staged entry is preserved for correct retry behavior. func TestNode_UnstageRetryKeepsStagedEntry(t *testing.T) { mi := newMockISCSIUtil() mm := newMockMountUtil() mm.unmountErr = errors.New("device busy") ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "busy-vol": { iqn: "iqn.2024.com.seaweedfs:busy-vol", iscsiAddr: "10.0.0.5:3260", isLocal: false, }, }, } stagingPath := t.TempDir() _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ VolumeId: "busy-vol", StagingTargetPath: stagingPath, }) if err == nil { t.Fatal("expected error from unmount failure") } // Staged entry should still be present for retry. ns.stagedMu.Lock() info, ok := ns.staged["busy-vol"] ns.stagedMu.Unlock() if !ok { t.Fatal("staged entry should be preserved on failure for retry") } if info.iqn != "iqn.2024.com.seaweedfs:busy-vol" { t.Fatalf("unexpected IQN: %s", info.iqn) } } // TestNode_ExpandVolume verifies happy path for NodeExpandVolume. func TestNode_ExpandVolume(t *testing.T) { ns, mi, mm := newTestNodeServer(t) stagingPath := t.TempDir() // Stage the volume first. _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Reset calls to track expand-specific calls. mi.calls = nil mm.calls = nil _, err = ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ VolumeId: "test-vol", CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, }) if err != nil { t.Fatalf("NodeExpandVolume: %v", err) } // Verify rescan was called. foundRescan := false for _, c := range mi.calls { if strings.HasPrefix(c, "rescan:") { foundRescan = true } } if !foundRescan { t.Fatalf("expected rescan call, got: %v", mi.calls) } // Verify resize was called. foundResize := false for _, c := range mm.calls { if strings.HasPrefix(c, "resizefs:") { foundResize = true } } if !foundResize { t.Fatalf("expected resizefs call, got: %v", mm.calls) } } // TestNode_ExpandNotStaged verifies error when volume is not staged. func TestNode_ExpandNotStaged(t *testing.T) { ns, _, _ := newTestNodeServer(t) _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ VolumeId: "not-staged", CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, }) if err == nil { t.Fatal("expected error for not-staged volume") } st, _ := status.FromError(err) if st.Code() != codes.FailedPrecondition { t.Fatalf("expected FailedPrecondition, got %v", st.Code()) } } // TestNode_ExpandMissingVolumeID verifies error for missing volume ID. func TestNode_ExpandMissingVolumeID(t *testing.T) { ns, _, _ := newTestNodeServer(t) _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{}) if err == nil { t.Fatal("expected error for missing volume ID") } st, _ := status.FromError(err) if st.Code() != codes.InvalidArgument { t.Fatalf("expected InvalidArgument, got %v", st.Code()) } } // TestNode_ExpandXFSUsesMountPath verifies xfs_growfs uses mount path. func TestNode_ExpandXFSUsesMountPath(t *testing.T) { mi := newMockISCSIUtil() mi.getDeviceResult = "/dev/sdb" mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "xfs-vol": { iqn: "iqn.2024.com.seaweedfs:xfs-vol", iscsiAddr: "10.0.0.5:3260", isLocal: false, fsType: "xfs", stagingPath: "/staging/xfs-vol", }, }, } // Mark as logged in so the check passes. mi.loggedIn["iqn.2024.com.seaweedfs:xfs-vol"] = true _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ VolumeId: "xfs-vol", CapacityRange: &csi.CapacityRange{RequiredBytes: 16 * 1024 * 1024}, }) if err != nil { t.Fatalf("NodeExpandVolume: %v", err) } // Verify resizefs was called with xfs type. foundXFS := false for _, c := range mm.calls { if strings.Contains(c, "resizefs:") && strings.Contains(c, ":xfs") { foundXFS = true // Verify mount path is used (not device path). if !strings.Contains(c, "/staging/xfs-vol") { t.Fatalf("xfs_growfs should use mount path, got: %s", c) } } } if !foundXFS { t.Fatalf("expected resizefs call with xfs, got: %v", mm.calls) } } // TestNode_ExpandRejectsNoISCSISession verifies error when iSCSI not logged in. func TestNode_ExpandRejectsNoISCSISession(t *testing.T) { mi := newMockISCSIUtil() mm := newMockMountUtil() ns := &nodeServer{ mgr: nil, nodeID: "test-node-1", iqnPrefix: "iqn.2024.com.seaweedfs", iscsiUtil: mi, mountUtil: mm, logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), staged: map[string]*stagedVolumeInfo{ "no-session-vol": { iqn: "iqn.2024.com.seaweedfs:no-session-vol", iscsiAddr: "10.0.0.5:3260", isLocal: false, fsType: "ext4", stagingPath: "/staging/no-session-vol", }, }, } // NOT logged in (loggedIn map is empty). _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ VolumeId: "no-session-vol", }) if err == nil { t.Fatal("expected error when iSCSI session not active") } st, _ := status.FromError(err) if st.Code() != codes.FailedPrecondition { t.Fatalf("expected FailedPrecondition, got %v", st.Code()) } } // TestNode_NodeGetCapabilities verifies EXPAND_VOLUME capability is present. func TestNode_NodeGetCapabilities(t *testing.T) { ns, _, _ := newTestNodeServer(t) resp, err := ns.NodeGetCapabilities(context.Background(), &csi.NodeGetCapabilitiesRequest{}) if err != nil { t.Fatalf("NodeGetCapabilities: %v", err) } hasStage := false hasExpand := false for _, cap := range resp.Capabilities { rpc := cap.GetRpc() if rpc == nil { continue } switch rpc.Type { case csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME: hasStage = true case csi.NodeServiceCapability_RPC_EXPAND_VOLUME: hasExpand = true } } if !hasStage { t.Fatal("expected STAGE_UNSTAGE_VOLUME capability") } if !hasExpand { t.Fatal("expected EXPAND_VOLUME capability") } } // TestNode_StageCapturesFSType verifies that fsType is captured in staged info. func TestNode_StageCapturesFSType(t *testing.T) { ns, _, _ := newTestNodeServer(t) stagingPath := t.TempDir() _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: &csi.VolumeCapability{ AccessType: &csi.VolumeCapability_Mount{ Mount: &csi.VolumeCapability_MountVolume{FsType: "xfs"}, }, }, }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } ns.stagedMu.Lock() info := ns.staged["test-vol"] ns.stagedMu.Unlock() if info == nil { t.Fatal("expected test-vol in staged map") } if info.fsType != "xfs" { t.Fatalf("fsType: got %q, want xfs", info.fsType) } if info.stagingPath != stagingPath { t.Fatalf("stagingPath: got %q, want %q", info.stagingPath, stagingPath) } } // TestNode_StageFallbackLocal verifies local fallback when no volume_context is provided. func TestNode_StageFallbackLocal(t *testing.T) { ns, mi, _ := newTestNodeServer(t) stagingPath := t.TempDir() // Stage without volume_context — should use local VolumeManager. _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: "test-vol", StagingTargetPath: stagingPath, VolumeCapability: testVolCap(), }) if err != nil { t.Fatalf("NodeStageVolume: %v", err) } // Should have called discovery with local addr. if len(mi.calls) < 1 { t.Fatal("expected iSCSI calls") } if mi.calls[0] != "discovery:"+ns.mgr.ListenAddr() { t.Fatalf("expected local discovery, got %q", mi.calls[0]) } // Verify staged as local. ns.stagedMu.Lock() info := ns.staged["test-vol"] ns.stagedMu.Unlock() if info == nil || !info.isLocal { t.Fatal("expected isLocal=true for local fallback") } }