Files
seaweedfs/weed/storage/blockvol/csi/controller.go
T
Ping QiuandClaude Opus 4.6 8b2b5f6f66 feat: Phase 6 CP6-3 -- failover + rebuild in Kubernetes, 126 tests
Wire low-level fencing primitives to master/VS control plane and CSI:

- Proto: replica/rebuild address fields on assignment/info/response messages
- Assignment queue: retain-until-confirmed (Peek+Confirm), stale epoch pruning
- VS assignment receiver: processes assignments from HeartbeatResponse
- BlockService replication: ProcessAssignments, deterministic ports (FNV hash)
- Registry replica tracking: SetReplica/ClearReplica/SwapPrimaryReplica
- CreateBlockVolume: primary + replica, enqueues assignments, single-copy mode
- Failover: lease-aware promotion, deferred timers with cancellation on reconnect
- ControllerPublish: returns fresh primary iSCSI address after failover
- Recovery: recoverBlockVolumes drains pendingRebuilds, enqueues Rebuilding
- Real integration tests on M02: failover address switch, rebuild data
  consistency, full lifecycle failover+rebuild (3 tests, all PASS)

Review fixes (12 findings, 5 High, 5 Medium, 2 Low):
- R1-1: AllocateBlockVolume returns replication ports
- R1-2: setupPrimaryReplication starts rebuild server
- R1-3: VS sends periodic block heartbeat for assignment confirmation
- R2-F1: LastLeaseGrant set before Register (no stale-lease race)
- R2-F2: Deferred promotion timers cancelled on VS reconnect
- R2-F3: SwapPrimaryReplica uses RoleToWire instead of uint32(1)
- R2-F4: DeleteBlockVolume deletes replica (best-effort)
- R2-F5: SwapPrimaryReplica computes epoch atomically under lock
- QA: SetReplica removes old replica from byServer index (BUG-QA-CP63-1)

126 CP6-3 tests (67 dev + 48 QA + 8 integration + 3 real).
Cumulative Phase 6: 352 tests. All PASS.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 00:52:05 -08:00

167 lines
5.5 KiB
Go

package csi
import (
"context"
"errors"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
defaultVolumeSizeBytes = 1 << 30 // 1 GiB
minVolumeSizeBytes = 1 << 20 // 1 MiB
blockSize = 4096
)
type controllerServer struct {
csi.UnimplementedControllerServer
backend VolumeBackend
}
func (s *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
if req.Name == "" {
return nil, status.Error(codes.InvalidArgument, "volume name is required")
}
if len(req.VolumeCapabilities) == 0 {
return nil, status.Error(codes.InvalidArgument, "volume capabilities are required")
}
sizeBytes := int64(defaultVolumeSizeBytes)
if req.CapacityRange != nil && req.CapacityRange.RequiredBytes > 0 {
sizeBytes = req.CapacityRange.RequiredBytes
} else if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 {
// No RequiredBytes set — use LimitBytes as the target size.
sizeBytes = req.CapacityRange.LimitBytes
}
if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 {
if req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes {
return nil, status.Errorf(codes.InvalidArgument,
"required_bytes (%d) exceeds limit_bytes (%d)",
req.CapacityRange.RequiredBytes, req.CapacityRange.LimitBytes)
}
}
if sizeBytes < minVolumeSizeBytes {
sizeBytes = minVolumeSizeBytes
}
// Round up to block size.
if sizeBytes%blockSize != 0 {
sizeBytes = (sizeBytes/blockSize + 1) * blockSize
}
// Verify rounded size still respects LimitBytes.
if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 {
if sizeBytes > req.CapacityRange.LimitBytes {
return nil, status.Errorf(codes.InvalidArgument,
"volume size (%d) after rounding exceeds limit_bytes (%d)",
sizeBytes, req.CapacityRange.LimitBytes)
}
}
info, err := s.backend.CreateVolume(context.Background(), req.Name, uint64(sizeBytes))
if err != nil {
if errors.Is(err, ErrVolumeSizeMismatch) {
return nil, status.Errorf(codes.AlreadyExists, "volume %q exists with different size", req.Name)
}
return nil, status.Errorf(codes.Internal, "create volume: %v", err)
}
resp := &csi.CreateVolumeResponse{
Volume: &csi.Volume{
VolumeId: info.VolumeID,
CapacityBytes: int64(info.CapacityBytes),
},
}
// Attach volume_context with iSCSI target info for NodeStageVolume.
if info.ISCSIAddr != "" || info.IQN != "" {
resp.Volume.VolumeContext = map[string]string{
"iscsiAddr": info.ISCSIAddr,
"iqn": info.IQN,
}
}
return resp, nil
}
func (s *controllerServer) DeleteVolume(_ context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
}
// Idempotent: DeleteVolume succeeds even if volume doesn't exist.
if err := s.backend.DeleteVolume(context.Background(), req.VolumeId); err != nil {
return nil, status.Errorf(codes.Internal, "delete volume: %v", err)
}
return &csi.DeleteVolumeResponse{}, nil
}
func (s *controllerServer) ControllerPublishVolume(_ context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
}
if req.NodeId == "" {
return nil, status.Error(codes.InvalidArgument, "node ID is required")
}
info, err := s.backend.LookupVolume(context.Background(), req.VolumeId)
if err != nil {
return nil, status.Errorf(codes.NotFound, "volume %q not found: %v", req.VolumeId, err)
}
return &csi.ControllerPublishVolumeResponse{
PublishContext: map[string]string{
"iscsiAddr": info.ISCSIAddr,
"iqn": info.IQN,
},
}, nil
}
func (s *controllerServer) ControllerUnpublishVolume(_ context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
}
// No-op: RWO enforced by iSCSI initiator single-login.
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
func (s *controllerServer) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
return &csi.ControllerGetCapabilitiesResponse{
Capabilities: []*csi.ControllerServiceCapability{
{
Type: &csi.ControllerServiceCapability_Rpc{
Rpc: &csi.ControllerServiceCapability_RPC{
Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
},
},
},
{
Type: &csi.ControllerServiceCapability_Rpc{
Rpc: &csi.ControllerServiceCapability_RPC{
Type: csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
},
},
},
},
}, nil
}
func (s *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
}
if len(req.VolumeCapabilities) == 0 {
return nil, status.Error(codes.InvalidArgument, "volume capabilities are required")
}
if _, err := s.backend.LookupVolume(context.Background(), req.VolumeId); err != nil {
return nil, status.Errorf(codes.NotFound, "volume %q not found", req.VolumeId)
}
return &csi.ValidateVolumeCapabilitiesResponse{
Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
VolumeCapabilities: req.VolumeCapabilities,
},
}, nil
}