Files
seaweedfs/weed/storage/blockvol/csi/controller_test.go
T
Ping QiuandClaude Opus 4.6 8b2b5f6f66 feat: Phase 6 CP6-3 -- failover + rebuild in Kubernetes, 126 tests
Wire low-level fencing primitives to master/VS control plane and CSI:

- Proto: replica/rebuild address fields on assignment/info/response messages
- Assignment queue: retain-until-confirmed (Peek+Confirm), stale epoch pruning
- VS assignment receiver: processes assignments from HeartbeatResponse
- BlockService replication: ProcessAssignments, deterministic ports (FNV hash)
- Registry replica tracking: SetReplica/ClearReplica/SwapPrimaryReplica
- CreateBlockVolume: primary + replica, enqueues assignments, single-copy mode
- Failover: lease-aware promotion, deferred timers with cancellation on reconnect
- ControllerPublish: returns fresh primary iSCSI address after failover
- Recovery: recoverBlockVolumes drains pendingRebuilds, enqueues Rebuilding
- Real integration tests on M02: failover address switch, rebuild data
  consistency, full lifecycle failover+rebuild (3 tests, all PASS)

Review fixes (12 findings, 5 High, 5 Medium, 2 Low):
- R1-1: AllocateBlockVolume returns replication ports
- R1-2: setupPrimaryReplication starts rebuild server
- R1-3: VS sends periodic block heartbeat for assignment confirmation
- R2-F1: LastLeaseGrant set before Register (no stale-lease race)
- R2-F2: Deferred promotion timers cancelled on VS reconnect
- R2-F3: SwapPrimaryReplica uses RoleToWire instead of uint32(1)
- R2-F4: DeleteBlockVolume deletes replica (best-effort)
- R2-F5: SwapPrimaryReplica computes epoch atomically under lock
- QA: SetReplica removes old replica from byServer index (BUG-QA-CP63-1)

126 CP6-3 tests (67 dev + 48 QA + 8 integration + 3 real).
Cumulative Phase 6: 352 tests. All PASS.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 00:52:05 -08:00

254 lines
6.7 KiB
Go

package csi
import (
"context"
"testing"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// testVolCaps returns a standard volume capability for testing.
func testVolCaps() []*csi.VolumeCapability {
return []*csi.VolumeCapability{{
AccessType: &csi.VolumeCapability_Mount{
Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"},
},
AccessMode: &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
},
}}
}
func testVolCap() *csi.VolumeCapability {
return testVolCaps()[0]
}
func TestController_CreateVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "test-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{
RequiredBytes: 4 * 1024 * 1024,
},
})
if err != nil {
t.Fatalf("CreateVolume: %v", err)
}
if resp.Volume.VolumeId != "test-vol" {
t.Fatalf("volume_id: got %q, want %q", resp.Volume.VolumeId, "test-vol")
}
if resp.Volume.CapacityBytes != 4*1024*1024 {
t.Fatalf("capacity: got %d, want %d", resp.Volume.CapacityBytes, 4*1024*1024)
}
if !mgr.VolumeExists("test-vol") {
t.Fatal("expected volume to exist")
}
// Verify volume_context has iSCSI info.
if resp.Volume.VolumeContext == nil {
t.Fatal("expected volume_context to be set")
}
if resp.Volume.VolumeContext["iqn"] == "" {
t.Fatal("expected iqn in volume_context")
}
}
func TestController_CreateIdempotent(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
req := &csi.CreateVolumeRequest{
Name: "idem-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{
RequiredBytes: 4 * 1024 * 1024,
},
}
if _, err := cs.CreateVolume(context.Background(), req); err != nil {
t.Fatalf("first create: %v", err)
}
// Second create with same size should succeed (idempotent).
resp, err := cs.CreateVolume(context.Background(), req)
if err != nil {
t.Fatalf("second create: %v", err)
}
if resp.Volume.VolumeId != "idem-vol" {
t.Fatalf("volume_id: got %q, want %q", resp.Volume.VolumeId, "idem-vol")
}
}
func TestController_DeleteVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Create then delete.
_, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "del-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{
RequiredBytes: 4 * 1024 * 1024,
},
})
if err != nil {
t.Fatalf("create: %v", err)
}
_, err = cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{
VolumeId: "del-vol",
})
if err != nil {
t.Fatalf("delete: %v", err)
}
if mgr.VolumeExists("del-vol") {
t.Fatal("expected volume to not exist after delete")
}
}
func TestController_DeleteNotFound(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Delete non-existent volume -- should succeed (CSI spec idempotency).
_, err := cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{
VolumeId: "nonexistent",
})
if err != nil {
t.Fatalf("delete non-existent: %v", err)
}
}
func TestControllerPublish_HappyPath(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Create a volume first.
mgr.CreateVolume("pub-vol", 4*1024*1024)
resp, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
VolumeId: "pub-vol",
NodeId: "node-1",
})
if err != nil {
t.Fatalf("ControllerPublishVolume: %v", err)
}
if resp.PublishContext == nil {
t.Fatal("expected publish_context")
}
if resp.PublishContext["iscsiAddr"] == "" {
t.Fatal("expected iscsiAddr in publish_context")
}
if resp.PublishContext["iqn"] == "" {
t.Fatal("expected iqn in publish_context")
}
}
func TestControllerPublish_MissingVolumeID(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
NodeId: "node-1",
})
if err == nil {
t.Fatal("expected error for missing volume ID")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
func TestControllerPublish_MissingNodeID(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
VolumeId: "vol1",
})
if err == nil {
t.Fatal("expected error for missing node ID")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
func TestControllerPublish_NotFound(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerPublishVolume(context.Background(), &csi.ControllerPublishVolumeRequest{
VolumeId: "nonexistent",
NodeId: "node-1",
})
if err == nil {
t.Fatal("expected error for not found")
}
st, _ := status.FromError(err)
if st.Code() != codes.NotFound {
t.Fatalf("expected NotFound, got %v", st.Code())
}
}
func TestControllerUnpublish_Success(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerUnpublishVolume(context.Background(), &csi.ControllerUnpublishVolumeRequest{
VolumeId: "any-vol",
NodeId: "node-1",
})
if err != nil {
t.Fatalf("ControllerUnpublishVolume: %v", err)
}
}
func TestController_Capabilities_IncludesPublish(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
resp, err := cs.ControllerGetCapabilities(context.Background(), &csi.ControllerGetCapabilitiesRequest{})
if err != nil {
t.Fatalf("ControllerGetCapabilities: %v", err)
}
hasCreate := false
hasPublish := false
for _, cap := range resp.Capabilities {
rpc := cap.GetRpc()
if rpc == nil {
continue
}
switch rpc.Type {
case csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME:
hasCreate = true
case csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME:
hasPublish = true
}
}
if !hasCreate {
t.Fatal("expected CREATE_DELETE_VOLUME capability")
}
if !hasPublish {
t.Fatal("expected PUBLISH_UNPUBLISH_VOLUME capability")
}
}